Analysis of False Belief Task
Sean Trott
April 24, 2025
Load LLM data
TODO: Load a separate .csv file with
information about number of parameters, amount of training data, base
vs. instruct.
# setwd("/Users/seantrott/Dropbox/UCSD/Research/NLMs/open_llm_tom/src/analysis")
directory_path <- "../../data/processed/fb_all/"
csv_files <- list.files(path = directory_path, pattern = "*.csv", full.names = TRUE)
csv_list <- csv_files %>%
map(~ read_csv(.))## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 0 Columns: 2
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (2): model_path, model_shorthand
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 0 Columns: 2
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (2): model_path, model_shorthand
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): lp_start, lp_end, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): lp_start, lp_end, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): lp_start, lp_end, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): lp_start, lp_end, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): lp_start, lp_end, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): lp_start, lp_end, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): lp_start, lp_end, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): lp_start, lp_end, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): lp_start, lp_end, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): lp_start, lp_end, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): lp_start, lp_end, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): lp_start, lp_end, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): lp_start, lp_end, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): lp_start, lp_end, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
## Rows: 192 Columns: 12
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): passage, start, end, knowledge_cue, first_mention, recent_mention, ...
## dbl (3): start_prob, end_prob, log_odds
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.df_all_models <- bind_rows(csv_list) %>%
mutate(model_shorthand = str_to_title(model_shorthand))
nrow(df_all_models)## [1] 10560table(df_all_models$model_path)##
## allenai/OLMo-2-0325-32B allenai/OLMo-2-0325-32B-DPO
## 192 192
## allenai/OLMo-2-0325-32B-Instruct allenai/OLMo-2-0325-32B-SFT
## 192 192
## allenai/OLMo-2-0425-1B allenai/OLMo-2-0425-1B-DPO
## 192 192
## allenai/OLMo-2-0425-1B-Instruct allenai/OLMo-2-0425-1B-SFT
## 192 192
## allenai/OLMo-2-1124-13B allenai/OLMo-2-1124-13B-DPO
## 192 192
## allenai/OLMo-2-1124-13B-Instruct allenai/OLMo-2-1124-13B-SFT
## 192 192
## allenai/OLMo-2-1124-7B allenai/OLMo-2-1124-7B-DPO
## 192 192
## allenai/OLMo-2-1124-7B-Instruct allenai/OLMo-2-1124-7B-SFT
## 192 192
## EleutherAI/pythia-1.4b EleutherAI/pythia-12b
## 192 192
## EleutherAI/pythia-14m EleutherAI/pythia-160m
## 192 192
## EleutherAI/pythia-1b EleutherAI/pythia-2.8b
## 192 192
## EleutherAI/pythia-410m EleutherAI/pythia-6.9b
## 192 192
## EleutherAI/pythia-70m google/gemma-2b
## 192 192
## google/gemma-2b-it meta-llama/Llama-2-13b-chat-hf
## 192 192
## meta-llama/Llama-2-13b-hf meta-llama/Llama-2-70b-chat-hf
## 192 192
## meta-llama/Llama-2-70b-hf meta-llama/Llama-2-7b-chat-hf
## 192 192
## meta-llama/Llama-2-7b-hf meta-llama/Llama-3.1-70B
## 192 192
## meta-llama/Llama-3.1-70B-Instruct meta-llama/Llama-3.1-8B-Instruct
## 192 192
## meta-llama/Meta-Llama-3-70B meta-llama/Meta-Llama-3-70B-Instruct
## 192 192
## meta-llama/Meta-Llama-3-8B meta-llama/Meta-Llama-3-8B-Instruct
## 192 192
## mistralai/Mixtral-8x7B-Instruct-v0.1 mistralai/Mixtral-8x7B-v0.1
## 192 192
## Qwen/Qwen2.5-0.5B Qwen/Qwen2.5-0.5B-Instruct
## 192 192
## Qwen/Qwen2.5-1.5B Qwen/Qwen2.5-1.5B-Instruct
## 192 192
## Qwen/Qwen2.5-14B Qwen/Qwen2.5-14B-Instruct
## 192 192
## Qwen/Qwen2.5-32B Qwen/Qwen2.5-32B-Instruct
## 192 192
## Qwen/Qwen2.5-3B Qwen/Qwen2.5-3B-Instruct
## 192 192
## Qwen/Qwen2.5-72B-Instruct Qwen/Qwen2.5-7B
## 192 192
## Qwen/Qwen2.5-7B-Instruct
## 192table(df_all_models$model_shorthand)##
## Gemma 2 2b Gemma 2 2b Instruct Llama 2 13b
## 192 192 192
## Llama 2 13b Instruct Llama 2 70b Llama 2 70b Instruct
## 192 192 192
## Llama 2 7b Llama 2 7b Instruct Llama 3 70b
## 192 192 192
## Llama 3 70b Instruct Llama 3 8b Llama 3 8b Instruct
## 192 192 192
## Llama 3.1 70b Llama 3.1 70b Instruct Llama 3.1 8b Instruct
## 192 192 192
## Mixtral 8x7b Mixtral 8x7b Instruct Olmo 2 13b
## 192 192 192
## Olmo 2 13b Dpo Olmo 2 13b Instruct Olmo 2 13b Sft
## 192 192 192
## Olmo 2 1b Olmo 2 1b Dpo Olmo 2 1b Instruct
## 192 192 192
## Olmo 2 1b Sft Olmo 2 32b Olmo 2 32b Dpo
## 192 192 192
## Olmo 2 32b Instruct Olmo 2 32b Sft Olmo 2 7b
## 192 192 192
## Olmo 2 7b Dpo Olmo 2 7b Instruct Olmo 2 7b Sft
## 192 192 192
## Pythia 1.4b Pythia 12b Pythia 14m
## 192 192 192
## Pythia 160m Pythia 1b Pythia 2.8b
## 192 192 192
## Pythia 410m Pythia 6.9b Pythia 70m
## 192 192 192
## Qwen 2.5 0.5b Qwen 2.5 0.5b Instruct Qwen 2.5 1.5 Instruct
## 192 192 192
## Qwen 2.5 1.5b Qwen 2.5 14b Qwen 2.5 14b Instruct
## 192 192 192
## Qwen 2.5 32b Qwen 2.5 32b Instruct Qwen 2.5 3b
## 192 192 192
## Qwen 2.5 3b Instruct Qwen 2.5 72b Instruct Qwen 2.5 7b
## 192 192 192
## Qwen 2.5 7b Instruct
## 192### Load #params
df_model_properties = read_csv("../../data/processed/model_properties.csv") %>%
mutate(model_shorthand = str_to_title(model_shorthand))## Rows: 58 Columns: 5
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (4): model_path, model_shorthand, base_instruct, model_family
## dbl (1): num_params
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.nrow(df_model_properties)## [1] 58### Load training data
df_training_data = read_csv("../../data/processed/model_training_data.csv")%>%
mutate(model_shorthand = str_to_title(model_shorthand))## Rows: 58 Columns: 4
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (3): model_path, model_shorthand, notes
## dbl (1): num_training_tokens
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.table(df_training_data$model_shorthand)##
## Gemma 2 2b Gemma 2 2b Instruct Gemma 2 7b
## 1 1 1
## Gemma 2 7b Instruct Llama 2 13b Llama 2 13b Instruct
## 1 1 1
## Llama 2 70b Llama 2 70b Instruct Llama 2 7b
## 1 1 1
## Llama 2 7b Instruct Llama 3 70b Llama 3 70b Instruct
## 1 1 1
## Llama 3 8b Llama 3 8b Instruct Llama 3.1 70b
## 1 1 1
## Llama 3.1 70b Instruct Llama 3.1 8b Llama 3.1 8b Instruct
## 1 1 1
## Mixtral 8x7b Mixtral 8x7b Instruct Olmo 2 13b
## 1 1 1
## Olmo 2 13b Dpo Olmo 2 13b Instruct Olmo 2 13b Sft
## 1 1 1
## Olmo 2 1b Olmo 2 1b Dpo Olmo 2 1b Instruct
## 1 1 1
## Olmo 2 1b Sft Olmo 2 32b Olmo 2 32b Dpo
## 1 1 1
## Olmo 2 32b Instruct Olmo 2 32b Sft Olmo 2 7b
## 1 1 1
## Olmo 2 7b Dpo Olmo 2 7b Instruct Olmo 2 7b Sft
## 1 1 1
## Pythia 1.4b Pythia 12b Pythia 14m
## 1 1 1
## Pythia 160m Pythia 1b Pythia 2.8b
## 1 1 1
## Pythia 410m Pythia 6.9b Pythia 70m
## 1 1 1
## Qwen 2.5 0.5b Qwen 2.5 0.5b Instruct Qwen 2.5 1.5 Instruct
## 1 1 1
## Qwen 2.5 1.5b Qwen 2.5 14b Qwen 2.5 14b Instruct
## 1 1 1
## Qwen 2.5 32b Qwen 2.5 32b Instruct Qwen 2.5 3b
## 1 1 1
## Qwen 2.5 3b Instruct Qwen 2.5 72b Instruct Qwen 2.5 7b
## 1 1 1
## Qwen 2.5 7b Instruct
## 1### Join
df_all_properties = df_training_data %>%
inner_join(df_model_properties)## Joining with `by = join_by(model_path, model_shorthand)`table(df_all_properties$model_shorthand)##
## Gemma 2 2b Gemma 2 2b Instruct Gemma 2 7b
## 1 1 1
## Gemma 2 7b Instruct Llama 2 13b Llama 2 13b Instruct
## 1 1 1
## Llama 2 70b Llama 2 70b Instruct Llama 2 7b
## 1 1 1
## Llama 2 7b Instruct Llama 3 70b Llama 3 70b Instruct
## 1 1 1
## Llama 3 8b Llama 3 8b Instruct Llama 3.1 70b
## 1 1 1
## Llama 3.1 70b Instruct Llama 3.1 8b Llama 3.1 8b Instruct
## 1 1 1
## Mixtral 8x7b Mixtral 8x7b Instruct Olmo 2 13b
## 1 1 1
## Olmo 2 13b Dpo Olmo 2 13b Instruct Olmo 2 13b Sft
## 1 1 1
## Olmo 2 1b Olmo 2 1b Dpo Olmo 2 1b Instruct
## 1 1 1
## Olmo 2 1b Sft Olmo 2 32b Olmo 2 32b Dpo
## 1 1 1
## Olmo 2 32b Instruct Olmo 2 32b Sft Olmo 2 7b
## 1 1 1
## Olmo 2 7b Dpo Olmo 2 7b Instruct Olmo 2 7b Sft
## 1 1 1
## Pythia 1.4b Pythia 12b Pythia 14m
## 1 1 1
## Pythia 160m Pythia 1b Pythia 2.8b
## 1 1 1
## Pythia 410m Pythia 6.9b Pythia 70m
## 1 1 1
## Qwen 2.5 0.5b Qwen 2.5 0.5b Instruct Qwen 2.5 1.5 Instruct
## 1 1 1
## Qwen 2.5 1.5b Qwen 2.5 14b Qwen 2.5 14b Instruct
## 1 1 1
## Qwen 2.5 32b Qwen 2.5 32b Instruct Qwen 2.5 3b
## 1 1 1
## Qwen 2.5 3b Instruct Qwen 2.5 72b Instruct Qwen 2.5 7b
## 1 1 1
## Qwen 2.5 7b Instruct
## 1nrow(df_all_properties)## [1] 58#### Bind with FB data
df_all_models = df_all_models %>%
inner_join(df_all_properties)## Joining with `by = join_by(model_path, model_shorthand)`nrow(df_all_models)## [1] 10560table(df_all_models$model_shorthand)##
## Gemma 2 2b Gemma 2 2b Instruct Llama 2 13b
## 192 192 192
## Llama 2 13b Instruct Llama 2 70b Llama 2 70b Instruct
## 192 192 192
## Llama 2 7b Llama 2 7b Instruct Llama 3 70b
## 192 192 192
## Llama 3 70b Instruct Llama 3 8b Llama 3 8b Instruct
## 192 192 192
## Llama 3.1 70b Llama 3.1 70b Instruct Llama 3.1 8b Instruct
## 192 192 192
## Mixtral 8x7b Mixtral 8x7b Instruct Olmo 2 13b
## 192 192 192
## Olmo 2 13b Dpo Olmo 2 13b Instruct Olmo 2 13b Sft
## 192 192 192
## Olmo 2 1b Olmo 2 1b Dpo Olmo 2 1b Instruct
## 192 192 192
## Olmo 2 1b Sft Olmo 2 32b Olmo 2 32b Dpo
## 192 192 192
## Olmo 2 32b Instruct Olmo 2 32b Sft Olmo 2 7b
## 192 192 192
## Olmo 2 7b Dpo Olmo 2 7b Instruct Olmo 2 7b Sft
## 192 192 192
## Pythia 1.4b Pythia 12b Pythia 14m
## 192 192 192
## Pythia 160m Pythia 1b Pythia 2.8b
## 192 192 192
## Pythia 410m Pythia 6.9b Pythia 70m
## 192 192 192
## Qwen 2.5 0.5b Qwen 2.5 0.5b Instruct Qwen 2.5 1.5 Instruct
## 192 192 192
## Qwen 2.5 1.5b Qwen 2.5 14b Qwen 2.5 14b Instruct
## 192 192 192
## Qwen 2.5 32b Qwen 2.5 32b Instruct Qwen 2.5 3b
## 192 192 192
## Qwen 2.5 3b Instruct Qwen 2.5 72b Instruct Qwen 2.5 7b
## 192 192 192
## Qwen 2.5 7b Instruct
## 192df_all_models = df_all_models %>%
mutate(model_shorthand = str_to_title(model_shorthand))
length(unique(df_all_models$model_shorthand))## [1] 55length(unique(df_all_models$model_family))## [1] 8LLM Analysis
Sensitivity to mental states
### Visualization
df_all_models %>%
ggplot(aes(x = log_odds,
y = reorder(model_shorthand, num_params),
fill = condition)) +
geom_density_ridges2(aes(height = ..density..),
color=NA,
scale=.85,
# size=1,
alpha = .8,
stat="density") +
labs(x = "Log Odds (Start vs. End)",
y = "",
fill = "") +
theme_minimal() +
geom_vline(xintercept = 0, linetype = "dotted") +
theme(
legend.position = "bottom"
) +
theme(axis.title = element_text(size=rel(1.2)),
axis.text = element_text(size = rel(1.2)),
legend.text = element_text(size = rel(1.2)),
legend.title = element_text(size = rel(1.2)),
strip.text.x = element_text(size = rel(1.2))) +
scale_fill_manual(values = viridisLite::viridis(2, option = "mako",
begin = 0.8, end = 0.15)) +
facet_wrap(~knowledge_cue)## Warning: The dot-dot notation (`..density..`) was deprecated in ggplot2 3.4.0.
## ℹ Please use `after_stat(density)` instead.
## This warning is displayed once every 8 hours.
## Call `lifecycle::last_lifecycle_warnings()` to see where this warning was
## generated.
### Visualization by model family
df_all_models %>%
group_by(passage, condition, knowledge_cue, model_family) %>%
summarise(m_lo = mean(log_odds)) %>%
ggplot(aes(x = m_lo,
y = model_family,
fill = condition)) +
geom_density_ridges2(aes(height = ..density..),
color=NA,
scale=.85,
# size=1,
alpha = .8,
stat="density") +
labs(x = "Log Odds (Start vs. End)",
y = "",
fill = "") +
theme_minimal() +
geom_vline(xintercept = 0, linetype = "dotted") +
theme(
legend.position = "bottom"
) +
theme(axis.title = element_text(size=rel(1.2)),
axis.text = element_text(size = rel(1.2)),
legend.text = element_text(size = rel(1.2)),
legend.title = element_text(size = rel(1.2)),
strip.text.x = element_text(size = rel(1.2))) +
scale_fill_manual(values = viridisLite::viridis(2, option = "mako",
begin = 0.8, end = 0.15)) +
facet_wrap(~knowledge_cue)## `summarise()` has grouped output by 'passage', 'condition', 'knowledge_cue'.
## You can override using the `.groups` argument.
### Visualization altogether
df_all_models %>%
group_by(passage, condition, knowledge_cue) %>%
summarise(m_lo = mean(log_odds)) %>%
ggplot(aes(x = m_lo,
y = knowledge_cue,
fill = condition)) +
geom_density_ridges2(aes(height = ..density..),
color=NA,
scale=.85,
# size=1,
alpha = .8,
stat="density") +
labs(x = "Log Odds (Start vs. End)",
y = "",
fill = "") +
theme_minimal() +
geom_vline(xintercept = 0, linetype = "dotted") +
theme(
legend.position = "bottom"
) +
theme(axis.title = element_text(size=rel(1.2)),
axis.text = element_text(size = rel(1.2)),
legend.text = element_text(size = rel(1.2)),
legend.title = element_text(size = rel(1.2)),
strip.text.x = element_text(size = rel(1.2))) +
scale_fill_manual(values = viridisLite::viridis(2, option = "mako",
begin = 0.8, end = 0.15))## `summarise()` has grouped output by 'passage', 'condition'. You can override
## using the `.groups` argument.
Each model on its own:
# Function to fit models and perform model comparison
compare_models <- function(df) {
if (n_distinct(df$condition) < 2) return(NULL) # Skip if only one condition
mod_full <- tryCatch(
lmer(log_odds ~ condition + knowledge_cue + first_mention + recent_mention +
(1 + condition | start), data = df, REML = FALSE),
error = function(e) NULL
)
mod_reduced <- tryCatch(
lmer(log_odds ~ knowledge_cue + first_mention + recent_mention +
(1 + condition | start), data = df, REML = FALSE),
error = function(e) NULL
)
if (is.null(mod_full) || is.null(mod_reduced)) return(NULL)
anova_result <- anova(mod_full, mod_reduced)
delta_aic <- AIC(mod_reduced) - AIC(mod_full)
lrt_stat <- anova_result$Chisq[2]
p_val <- anova_result$`Pr(>Chisq)`[2]
# Extract coefficients
coefs <- tryCatch(fixef(mod_full), error = function(e) return(rep(NA, 2)))
cond_coef <- coefs[grep("^condition", names(coefs))]
cue_coef <- coefs[grep("^knowledge_cue", names(coefs))]
tibble(
model_path = unique(df$model_path),
delta_AIC = delta_aic,
LRT_stat = lrt_stat,
p_value = p_val,
condition_coef = cond_coef,
knowledge_cue_coef = cue_coef
)
}
# Apply to each model_path
results_by_model_path <- df_all_models %>%
group_by(model_path) %>%
group_split() %>%
map_dfr(compare_models)## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')results_by_model_path %>%
arrange(LRT_stat)## # A tibble: 55 × 6
## model_path delta_AIC LRT_stat p_value condition_coef knowledge_cue_coef
## <chr> <dbl> <dbl> <dbl> <dbl> <dbl>
## 1 meta-llama/Meta… -2.00 7.77e-5 0.993 -0.000782 0.00235
## 2 EleutherAI/pyth… -2.00 4.77e-4 0.983 0.00338 -0.240
## 3 meta-llama/Meta… -2.00 2.58e-3 0.959 -0.00507 0.0144
## 4 Qwen/Qwen2.5-1.… -1.99 6.02e-3 0.938 0.0329 -0.147
## 5 meta-llama/Llam… -1.99 7.80e-3 0.930 -0.0446 -1.15
## 6 Qwen/Qwen2.5-0.… -1.99 8.76e-3 0.925 0.0348 -0.568
## 7 allenai/OLMo-2-… -1.99 1.05e-2 0.918 -0.0443 1.06
## 8 google/gemma-2b -1.98 1.54e-2 0.901 -0.0177 -0.103
## 9 EleutherAI/pyth… -1.98 2.07e-2 0.886 -0.0298 0.464
## 10 allenai/OLMo-2-… -1.98 2.18e-2 0.883 0.102 2.32
## # ℹ 45 more rowssummary(results_by_model_path$LRT_stat)## Min. 1st Qu. Median Mean 3rd Qu. Max.
## 7.770e-05 2.230e-01 3.300e+00 7.482e+00 1.428e+01 2.546e+01summary(results_by_model_path$condition_coef)## Min. 1st Qu. Median Mean 3rd Qu. Max.
## -6.9767 -2.9649 -0.8921 -1.6393 0.0013 0.7203summary(results_by_model_path$knowledge_cue_coef)## Min. 1st Qu. Median Mean 3rd Qu. Max.
## -9.2877 -2.1903 -1.0411 -1.4878 -0.1936 2.9367### TODO: Should we adjust? What hypothesis are we testing?
results_by_model_path$p_adj = p.adjust(results_by_model_path$p_value, method = "holm")
results_by_model_path = results_by_model_path %>%
mutate(sig = p_adj < .05)
mean(results_by_model_path$sig)## [1] 0.3636364All models together:
### Mixed models
mod_full = lmer(data = df_all_models,
log_odds ~ condition + knowledge_cue + first_mention + recent_mention +
(1 + condition | model_path) + (1 + condition | start),
REML = FALSE)
mod_reduced = lmer(data = df_all_models,
log_odds ~ knowledge_cue + first_mention + recent_mention +
(1 + condition | model_path) + (1 + condition | start),
REML = FALSE)
summary(mod_full)## Linear mixed model fit by maximum likelihood . t-tests use Satterthwaite's
## method [lmerModLmerTest]
## Formula:
## log_odds ~ condition + knowledge_cue + first_mention + recent_mention +
## (1 + condition | model_path) + (1 + condition | start)
## Data: df_all_models
##
## AIC BIC logLik -2*log(L) df.resid
## 59398.1 59485.3 -29687.0 59374.1 10548
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -11.5087 -0.4004 0.0040 0.4091 7.5753
##
## Random effects:
## Groups Name Variance Std.Dev. Corr
## model_path (Intercept) 5.6724 2.3817
## conditionTrue Belief 3.9081 1.9769 -0.78
## start (Intercept) 0.5061 0.7114
## conditionTrue Belief 0.3079 0.5549 -0.30
## Residual 15.6299 3.9535
## Number of obs: 10560, groups: model_path, 55; start, 10
##
## Fixed effects:
## Estimate Std. Error df t value Pr(>|t|)
## (Intercept) 1.509e+00 4.017e-01 5.073e+01 3.756 0.000446 ***
## conditionTrue Belief -1.639e+00 3.289e-01 4.614e+01 -4.984 9.21e-06 ***
## knowledge_cueImplicit -1.488e+00 7.694e-02 1.043e+04 -19.336 < 2e-16 ***
## first_mentionStart 1.549e-01 7.694e-02 1.043e+04 2.013 0.044145 *
## recent_mentionStart 2.732e-01 7.694e-02 1.043e+04 3.551 0.000386 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) cndtTB knwl_I frst_S
## condtnTrBlf -0.619
## knwldg_cImp -0.096 0.000
## frst_mntnSt -0.096 0.000 0.000
## rcnt_mntnSt -0.096 0.000 0.000 0.000anova(mod_full, mod_reduced)## Data: df_all_models
## Models:
## mod_reduced: log_odds ~ knowledge_cue + first_mention + recent_mention + (1 + condition | model_path) + (1 + condition | start)
## mod_full: log_odds ~ condition + knowledge_cue + first_mention + recent_mention + (1 + condition | model_path) + (1 + condition | start)
## npar AIC BIC logLik -2*log(L) Chisq Df Pr(>Chisq)
## mod_reduced 11 59414 59494 -29696 59392
## mod_full 12 59398 59485 -29687 59374 18.172 1 2.018e-05 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1Accuracy metric
df_all_models = df_all_models %>%
mutate(correct = case_when(
condition == "False Belief" & log_odds > 0 ~ TRUE,
condition == "True Belief" & log_odds <= 0 ~ TRUE,
TRUE ~ FALSE # all other cases are incorrect
))
df_summ = df_all_models %>%
group_by(model_path, model_shorthand, model_family,
num_params, num_training_tokens, base_instruct) %>%
summarise(mean_accuracy = mean(correct))## `summarise()` has grouped output by 'model_path', 'model_shorthand',
## 'model_family', 'num_params', 'num_training_tokens'. You can override using the
## `.groups` argument.df_summ %>%
select(model_shorthand, mean_accuracy)## Adding missing grouping variables: `model_path`, `model_family`, `num_params`,
## `num_training_tokens`## # A tibble: 55 × 6
## # Groups: model_path, model_shorthand, model_family, num_params,
## # num_training_tokens [55]
## model_path model_family num_params num_training_tokens model_shorthand
## <chr> <chr> <dbl> <dbl> <chr>
## 1 EleutherAI/pythi… Pythia 1.52e 9 3 e11 Pythia 1.4b
## 2 EleutherAI/pythi… Pythia 1.20e10 3 e11 Pythia 12b
## 3 EleutherAI/pythi… Pythia 3.92e 7 3 e11 Pythia 14m
## 4 EleutherAI/pythi… Pythia 2.13e 8 3 e11 Pythia 160m
## 5 EleutherAI/pythi… Pythia 1.08e 9 3 e11 Pythia 1b
## 6 EleutherAI/pythi… Pythia 2.91e 9 3 e11 Pythia 2.8b
## 7 EleutherAI/pythi… Pythia 5.06e 8 3 e11 Pythia 410m
## 8 EleutherAI/pythi… Pythia 6.99e 9 3 e11 Pythia 6.9b
## 9 EleutherAI/pythi… Pythia 9.56e 7 3 e11 Pythia 70m
## 10 Qwen/Qwen2.5-0.5B Qwen 2.5 4.94e 8 1.80e13 Qwen 2.5 0.5b
## # ℹ 45 more rows
## # ℹ 1 more variable: mean_accuracy <dbl>mean(df_all_models$correct)## [1] 0.5733902### "Wisdom of the crowd"?
df_lo_avg = df_all_models %>%
group_by(passage, condition) %>%
summarise(m_lo = mean(log_odds)) %>%
mutate(correct = case_when(
condition == "False Belief" & m_lo > 0 ~ TRUE,
condition == "True Belief" & m_lo <= 0 ~ TRUE,
TRUE ~ FALSE # all other cases are incorrect
)) ## `summarise()` has grouped output by 'passage'. You can override using the
## `.groups` argument.mean(df_lo_avg$correct)## [1] 0.7291667df_summ %>%
ggplot(aes(x = num_params,
y = mean_accuracy,
color = model_family,
shape = base_instruct)) +
geom_point(size = 6,
alpha = .5) +
geom_hline(yintercept = .83,##TODO: Calculate from scratch
linetype = "dotted", color = "red",
size = 1.2, alpha = .8) +
geom_hline(yintercept = .9,##TODO: Calculate from scratch
linetype = "dotted", color = "blue",
size = 1.2, alpha = .8) +
geom_hline(yintercept = .5, linetype = "dotted",
size = 1.2, alpha = .5) +
scale_x_log10() +
# geom_text_repel(aes(label=model_shorthand), size=3) +
scale_y_continuous(limits = c(0, 1)) +
labs(x = "Parameters",
y = "Accuracy",
color = "",
shape = "") +
theme_minimal() +
scale_color_manual(values = viridisLite::viridis(8, option = "mako",
begin = 0.8, end = 0.15)) +
theme(text = element_text(size = 15),
legend.position="bottom")## Warning: Using `size` aesthetic for lines was deprecated in ggplot2 3.4.0.
## ℹ Please use `linewidth` instead.
## This warning is displayed once every 8 hours.
## Call `lifecycle::last_lifecycle_warnings()` to see where this warning was
## generated.
df_summ %>%
ggplot(aes(x = num_training_tokens,
y = mean_accuracy,
color = model_family,
shape = base_instruct)) +
geom_point(size = 6,
alpha = .5) +
geom_hline(yintercept = .83,##TODO: Calculate from scratch
linetype = "dotted", color = "red",
size = 1.2, alpha = .8) +
geom_hline(yintercept = .5, linetype = "dotted",
size = 1.2, alpha = .5) +
geom_hline(yintercept = .9,##TODO: Calculate from scratch
linetype = "dotted", color = "blue",
size = 1.2, alpha = .8) +
scale_x_log10() +
# geom_text_repel(aes(label=model_shorthand), size=3) +
scale_y_continuous(limits = c(0, 1)) +
labs(x = "#Training Tokens",
y = "Accuracy",
color = "",
shape = "") +
theme_minimal() +
scale_color_manual(values = viridisLite::viridis(8, option = "mako",
begin = 0.8, end = 0.15)) +
theme(text = element_text(size = 15),
legend.position="bottom")## Warning: Removed 2 rows containing missing values or values outside the scale range
## (`geom_point()`).
### How does model size predict accuracy?
mod_full = glmer(data = df_all_models,
correct ~ condition + knowledge_cue +
log10(num_params) + log10(num_training_tokens) + base_instruct +
(1 | start) + (1|model_family),
family = binomial())
summary(mod_full)## Generalized linear mixed model fit by maximum likelihood (Laplace
## Approximation) [glmerMod]
## Family: binomial ( logit )
## Formula:
## correct ~ condition + knowledge_cue + log10(num_params) + log10(num_training_tokens) +
## base_instruct + (1 | start) + (1 | model_family)
## Data: df_all_models
##
## AIC BIC logLik -2*log(L) df.resid
## 13704.0 13761.9 -6844.0 13688.0 10168
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -1.5581 -1.1117 0.7076 0.8610 1.4216
##
## Random effects:
## Groups Name Variance Std.Dev.
## start (Intercept) 0.01545 0.1243
## model_family (Intercept) 0.01355 0.1164
## Number of obs: 10176, groups: start, 10; model_family, 7
##
## Fixed effects:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) -3.496604 1.057875 -3.305 0.000949 ***
## conditionTrue Belief -0.185417 0.040539 -4.574 4.79e-06 ***
## knowledge_cueImplicit 0.087052 0.040534 2.148 0.031743 *
## log10(num_params) 0.281655 0.034341 8.202 2.37e-16 ***
## log10(num_training_tokens) 0.082718 0.086765 0.953 0.340410
## base_instructInstruct -0.005725 0.044662 -0.128 0.897997
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) cndtTB knwl_I l10(_) l10(__
## condtnTrBlf -0.016
## knwldg_cImp -0.021 -0.002
## lg10(nm_pr) -0.037 -0.008 0.004
## lg10(nm_t_) -0.949 -0.001 0.001 -0.272
## bs_nstrctIn 0.134 0.000 0.000 -0.021 -0.140Item-wise analysis
df_item_accuracy = df_all_models %>%
group_by(start) %>%
summarise(mean_accuracy = mean(correct)) %>%
arrange(mean_accuracy)
df_item_accuracy %>%
ggplot(aes(x = mean_accuracy)) +
geom_histogram(alpha = .5) +
labs(x = "Item-wise accuracy",
y = "Count",
color = "",
shape = "") +
theme_minimal() +
theme(text = element_text(size = 15),
legend.position="bottom")## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
Comparison to human baseline
Load human data
df_human = read_csv("../../data/processed/human/human_fb_cleaned.csv") %>%
select(participant_id, item_id, passage,
is_correct, is_start, is_end,
reaction_time, condition, response, first_mention, recent_mention, knowledge_cue)## New names:
## Rows: 613 Columns: 51
## ── Column specification
## ──────────────────────────────────────────────────────── Delimiter: "," chr
## (26): item_id, item_type, correct_answer, response, condition, first_men... dbl
## (21): ...1, id, participant_id, item, trial_index, reaction_time, trial_... lgl
## (4): is_correct, is_start, is_end, excluded.attention
## ℹ Use `spec()` to retrieve the full column specification for this data. ℹ
## Specify the column types or set `show_col_types = FALSE` to quiet this message.
## • `` -> `...1`mean(df_human$is_correct)## [1] 0.8270799df_by_item = df_human %>%
group_by(condition, passage, knowledge_cue) %>%
summarise(p_start = mean(is_start) + .0001,
p_end = mean(is_end) + .0001) %>%
mutate(log_odds = log((p_start / p_end))) %>%
mutate(model_shorthand = "Human") %>%
mutate(correct = case_when(
condition == "False Belief" & log_odds > 0 ~ TRUE,
condition == "True Belief" & log_odds <= 0 ~ TRUE,
TRUE ~ FALSE # all other cases are incorrect
))## `summarise()` has grouped output by 'condition', 'passage'. You can override
## using the `.groups` argument.### Mean using log-odds measure forh umans
mean(df_by_item$correct)## [1] 0.8972973### Merge
df_merged = df_all_models %>%
bind_rows(df_by_item)Recreate model-wise distributions
### Visualization
df_merged %>%
ggplot(aes(x = log_odds,
y = reorder(model_shorthand, num_params),
fill = condition)) +
geom_density_ridges2(aes(height = ..density..),
color=NA,
scale=.85,
# size=1,
alpha = .8,
stat="density") +
labs(x = "Log Odds (Start vs. End)",
y = "",
fill = "") +
theme_minimal() +
geom_vline(xintercept = 0, linetype = "dotted") +
theme(
legend.position = "bottom"
) +
theme(axis.title = element_text(size=rel(1.2)),
axis.text = element_text(size = rel(1.2)),
legend.text = element_text(size = rel(1.2)),
legend.title = element_text(size = rel(1.2)),
strip.text.x = element_text(size = rel(1.2))) +
scale_fill_manual(values = viridisLite::viridis(2, option = "mako",
begin = 0.8, end = 0.15)) +
facet_wrap(~knowledge_cue)
Correlation matrix
df_model_means = df_all_models %>%
group_by(passage, condition, knowledge_cue) %>%
summarise(log_odds = mean(log_odds)) %>%
mutate(model_shorthand = "All Models")## `summarise()` has grouped output by 'passage', 'condition'. You can override
## using the `.groups` argument.# df_merged = df_merged %>%
# bind_rows(df_model_means)
df_wide <- df_merged %>%
select(passage, model_shorthand, log_odds) %>%
pivot_wider(names_from = model_shorthand, values_from = log_odds)
cor_matrix <- df_wide %>%
select(-passage) %>%
cor(use = "pairwise.complete.obs")
cor_matrix## Gemma 2 2b Instruct Gemma 2 2b Llama 2 13b Instruct
## Gemma 2 2b Instruct 1.000000000 0.6722852228 -0.455377518
## Gemma 2 2b 0.672285223 1.0000000000 -0.136248828
## Llama 2 13b Instruct -0.455377518 -0.1362488281 1.000000000
## Llama 2 13b -0.430443081 -0.2095676535 0.875170547
## Llama 2 70b Instruct -0.567846375 -0.3249761320 0.819181540
## Llama 2 70b -0.526272683 -0.3250232879 0.747631965
## Llama 2 7b Instruct -0.487029077 -0.2728260654 0.867223907
## Llama 2 7b -0.455588599 -0.2336015524 0.809921808
## Llama 3.1 70b Instruct -0.257783261 -0.1614057153 0.334673906
## Llama 3.1 70b -0.447401985 -0.3528670325 0.410528002
## Llama 3.1 8b Instruct -0.123875595 -0.0860866978 0.520200932
## Llama 3 70b Instruct -0.233349901 -0.0999230900 0.366727102
## Llama 3 70b -0.467179646 -0.3421167591 0.440532537
## Llama 3 8b Instruct 0.376631659 0.0307171580 -0.363834021
## Llama 3 8b 0.204659447 -0.2704714772 -0.269952314
## Mixtral 8x7b Instruct -0.456066026 -0.2846145058 0.626521493
## Mixtral 8x7b -0.514902287 -0.3091203411 0.697092113
## Olmo 2 32b Dpo -0.268731991 -0.3276903734 0.312914871
## Olmo 2 32b Instruct -0.265877635 -0.3249597588 0.308630576
## Olmo 2 32b Sft -0.222031504 -0.2897456531 0.309346911
## Olmo 2 32b -0.259617495 -0.2950606285 0.345512286
## Olmo 2 1b Dpo -0.366871993 -0.1744724378 0.105188814
## Olmo 2 1b Instruct -0.310201372 -0.1403751566 0.073647917
## Olmo 2 1b Sft -0.429979033 -0.2509243090 0.159629937
## Olmo 2 1b 0.334363799 0.0282235873 -0.093465625
## Olmo 2 13b Dpo -0.055796231 0.0005462096 0.516853978
## Olmo 2 13b Instruct -0.053642530 -0.0032235768 0.512386340
## Olmo 2 13b Sft -0.106839894 -0.0225046719 0.543185459
## Olmo 2 13b -0.057371452 -0.0201578956 0.507096428
## Olmo 2 7b Dpo 0.012161608 0.0295096248 0.429992932
## Olmo 2 7b Instruct 0.012405072 0.0434048880 0.441618955
## Olmo 2 7b Sft 0.037875735 -0.0115691823 0.380957048
## Olmo 2 7b -0.031654087 -0.1675830716 0.362754895
## Pythia 1.4b 0.009603831 0.0241794124 0.268290222
## Pythia 12b 0.044907507 0.2179821432 0.465971881
## Pythia 14m -0.122810764 -0.0879402078 0.005675609
## Pythia 160m 0.270712660 0.0386396823 -0.182986484
## Pythia 1b -0.526743842 -0.2986455554 0.359508222
## Pythia 2.8b 0.110510135 0.2605135744 0.375653486
## Pythia 410m -0.395807007 0.0647016129 0.271912692
## Pythia 6.9b -0.041708526 -0.0494400439 0.279001416
## Pythia 70m 0.294286567 0.1237175419 -0.006264252
## Qwen 2.5 0.5b Instruct -0.013313842 -0.1695754385 0.092635402
## Qwen 2.5 0.5b -0.041815769 -0.1046034429 0.026696447
## Qwen 2.5 1.5 Instruct -0.232230721 -0.1671052932 0.385126424
## Qwen 2.5 1.5b -0.276793142 -0.1515846259 0.402595243
## Qwen 2.5 14b Instruct -0.038042581 -0.2176876162 0.030085275
## Qwen 2.5 14b -0.047246281 -0.2199546340 0.174146602
## Qwen 2.5 32b Instruct 0.083448171 0.0229545069 0.068610010
## Qwen 2.5 32b -0.003270134 0.0125410809 0.061451445
## Qwen 2.5 3b Instruct 0.003695353 0.0055565414 -0.032110903
## Qwen 2.5 3b -0.205550984 -0.1407393656 0.223711469
## Qwen 2.5 72b Instruct 0.238807508 0.2562909966 0.125489964
## Qwen 2.5 7b Instruct -0.158102063 -0.1049743214 0.254507331
## Qwen 2.5 7b 0.035990949 0.0585283926 0.147727211
## Human 0.028923543 0.0091288412 0.214424411
## Llama 2 13b Llama 2 70b Instruct Llama 2 70b
## Gemma 2 2b Instruct -0.430443081 -0.56784637 -0.5262726832
## Gemma 2 2b -0.209567653 -0.32497613 -0.3250232879
## Llama 2 13b Instruct 0.875170547 0.81918154 0.7476319648
## Llama 2 13b 1.000000000 0.81015621 0.8705342538
## Llama 2 70b Instruct 0.810156209 1.00000000 0.8540887683
## Llama 2 70b 0.870534254 0.85408877 1.0000000000
## Llama 2 7b Instruct 0.919057551 0.81709285 0.8178476195
## Llama 2 7b 0.960932122 0.79137968 0.8942068065
## Llama 3.1 70b Instruct 0.240569854 0.49875767 0.3190516687
## Llama 3.1 70b 0.345186408 0.57704564 0.4606029269
## Llama 3.1 8b Instruct 0.503729418 0.54197076 0.3851254949
## Llama 3 70b Instruct 0.251357132 0.48889958 0.3134812494
## Llama 3 70b 0.381960369 0.61558193 0.4987954501
## Llama 3 8b Instruct -0.415213609 -0.40096205 -0.4503173747
## Llama 3 8b -0.138375140 -0.25717891 -0.0417731546
## Mixtral 8x7b Instruct 0.622445639 0.59956069 0.5891233240
## Mixtral 8x7b 0.739906704 0.72378918 0.7357608660
## Olmo 2 32b Dpo 0.294794756 0.44682547 0.3766510528
## Olmo 2 32b Instruct 0.286943059 0.44270686 0.3649038224
## Olmo 2 32b Sft 0.320502690 0.44709231 0.3982369588
## Olmo 2 32b 0.340737501 0.46312550 0.4384398525
## Olmo 2 1b Dpo 0.049298814 0.18245561 0.1435890148
## Olmo 2 1b Instruct 0.039881865 0.13531670 0.1335709174
## Olmo 2 1b Sft 0.070487137 0.24185939 0.1567253049
## Olmo 2 1b 0.080015299 0.07394447 0.0908610060
## Olmo 2 13b Dpo 0.462989964 0.51736173 0.4140093953
## Olmo 2 13b Instruct 0.459307362 0.52323489 0.4169292244
## Olmo 2 13b Sft 0.506657649 0.53545905 0.4437003495
## Olmo 2 13b 0.429371098 0.48347493 0.3623477768
## Olmo 2 7b Dpo 0.363697138 0.37884347 0.3085271474
## Olmo 2 7b Instruct 0.368128440 0.38647439 0.3059019408
## Olmo 2 7b Sft 0.341651082 0.35708366 0.2817788508
## Olmo 2 7b 0.325594779 0.36848868 0.2701557493
## Pythia 1.4b 0.290153425 0.21330130 0.2073337766
## Pythia 12b 0.449597866 0.44915881 0.3591531898
## Pythia 14m 0.094687910 -0.06368786 0.2278739241
## Pythia 160m -0.148554379 -0.28843450 -0.1308181439
## Pythia 1b 0.307561976 0.43807501 0.3690260453
## Pythia 2.8b 0.370897011 0.36034944 0.3455654662
## Pythia 410m 0.136403077 0.16980963 0.1344582790
## Pythia 6.9b 0.245940355 0.27742412 0.2132555625
## Pythia 70m -0.037078049 -0.08178093 -0.0647124820
## Qwen 2.5 0.5b Instruct -0.010331259 0.08425918 -0.0343492902
## Qwen 2.5 0.5b -0.038862833 0.02590239 0.0121153038
## Qwen 2.5 1.5 Instruct 0.324765069 0.35181156 0.3531401774
## Qwen 2.5 1.5b 0.398027534 0.42323816 0.4226605129
## Qwen 2.5 14b Instruct 0.004240204 0.11689475 0.1043277964
## Qwen 2.5 14b 0.201610773 0.31136252 0.3194456778
## Qwen 2.5 32b Instruct -0.075367377 0.10300778 -0.0130232225
## Qwen 2.5 32b -0.061423269 0.16254795 0.0002291193
## Qwen 2.5 3b Instruct -0.178827516 -0.05293058 -0.1531463961
## Qwen 2.5 3b 0.079176267 0.19515132 0.0716659942
## Qwen 2.5 72b Instruct -0.021547644 0.03254169 -0.0931586339
## Qwen 2.5 7b Instruct 0.222673386 0.33877909 0.2917473682
## Qwen 2.5 7b 0.150201931 0.23983543 0.2052341271
## Human 0.113504588 0.18415995 0.0930685828
## Llama 2 7b Instruct Llama 2 7b Llama 3.1 70b Instruct
## Gemma 2 2b Instruct -0.487029077 -0.45558860 -0.25778326
## Gemma 2 2b -0.272826065 -0.23360155 -0.16140572
## Llama 2 13b Instruct 0.867223907 0.80992181 0.33467391
## Llama 2 13b 0.919057551 0.96093212 0.24056985
## Llama 2 70b Instruct 0.817092855 0.79137968 0.49875767
## Llama 2 70b 0.817847620 0.89420681 0.31905167
## Llama 2 7b Instruct 1.000000000 0.93351696 0.27182375
## Llama 2 7b 0.933516961 1.00000000 0.22610095
## Llama 3.1 70b Instruct 0.271823751 0.22610095 1.00000000
## Llama 3.1 70b 0.363911890 0.33293488 0.86085406
## Llama 3.1 8b Instruct 0.447291883 0.44633773 0.44355283
## Llama 3 70b Instruct 0.265855387 0.21747706 0.89807709
## Llama 3 70b 0.382693003 0.36271528 0.85468575
## Llama 3 8b Instruct -0.364370723 -0.42868174 -0.05080792
## Llama 3 8b -0.188857045 -0.11260934 -0.10972327
## Mixtral 8x7b Instruct 0.561777086 0.59063184 0.43324826
## Mixtral 8x7b 0.678456336 0.71674442 0.45913940
## Olmo 2 32b Dpo 0.269859017 0.28371613 0.58956655
## Olmo 2 32b Instruct 0.262343500 0.27620902 0.59172568
## Olmo 2 32b Sft 0.273323067 0.30718059 0.54846014
## Olmo 2 32b 0.299070472 0.32391648 0.55625015
## Olmo 2 1b Dpo 0.057318850 0.08737688 0.04034964
## Olmo 2 1b Instruct 0.041950545 0.08602484 -0.00434960
## Olmo 2 1b Sft 0.077973331 0.09842031 0.12281636
## Olmo 2 1b 0.068027782 0.10616224 0.09677086
## Olmo 2 13b Dpo 0.433407740 0.41047983 0.56104077
## Olmo 2 13b Instruct 0.429894259 0.40638425 0.57051664
## Olmo 2 13b Sft 0.484442279 0.46328999 0.50994483
## Olmo 2 13b 0.417944041 0.37640396 0.50917812
## Olmo 2 7b Dpo 0.325962764 0.31488415 0.36135596
## Olmo 2 7b Instruct 0.334462042 0.31755264 0.37069583
## Olmo 2 7b Sft 0.311106197 0.30245569 0.35063386
## Olmo 2 7b 0.325998013 0.27029108 0.29665598
## Pythia 1.4b 0.332839311 0.27264813 0.18467125
## Pythia 12b 0.521143932 0.45821914 0.18581031
## Pythia 14m 0.073199503 0.15196072 -0.14131014
## Pythia 160m -0.160759380 -0.14131209 -0.23249433
## Pythia 1b 0.304056219 0.27945097 0.35931081
## Pythia 2.8b 0.293641902 0.35271878 0.10894154
## Pythia 410m 0.160859247 0.12633337 0.07753735
## Pythia 6.9b 0.250557626 0.23662931 0.12055821
## Pythia 70m -0.006581317 -0.06250119 -0.18021780
## Qwen 2.5 0.5b Instruct 0.052594364 -0.06941880 0.06631334
## Qwen 2.5 0.5b 0.001633687 -0.05070757 0.02468580
## Qwen 2.5 1.5 Instruct 0.370550193 0.32652450 0.07294145
## Qwen 2.5 1.5b 0.424330917 0.39397662 0.16623809
## Qwen 2.5 14b Instruct -0.028440696 -0.01591284 0.40280713
## Qwen 2.5 14b 0.165129986 0.19818624 0.50802653
## Qwen 2.5 32b Instruct -0.046519454 -0.10240867 0.32372848
## Qwen 2.5 32b -0.036620673 -0.08815587 0.46809143
## Qwen 2.5 3b Instruct -0.149917176 -0.17523844 0.02442587
## Qwen 2.5 3b 0.107798227 0.05589994 0.15912046
## Qwen 2.5 72b Instruct -0.069362215 -0.07526131 0.50129587
## Qwen 2.5 7b Instruct 0.178713042 0.20465624 0.44063594
## Qwen 2.5 7b 0.071061723 0.13032897 0.35603632
## Human 0.108824844 0.07404699 0.46293384
## Llama 3.1 70b Llama 3.1 8b Instruct Llama 3 70b Instruct
## Gemma 2 2b Instruct -0.44740199 -0.123875595 -0.233349901
## Gemma 2 2b -0.35286703 -0.086086698 -0.099923090
## Llama 2 13b Instruct 0.41052800 0.520200932 0.366727102
## Llama 2 13b 0.34518641 0.503729418 0.251357132
## Llama 2 70b Instruct 0.57704564 0.541970765 0.488899576
## Llama 2 70b 0.46060293 0.385125495 0.313481249
## Llama 2 7b Instruct 0.36391189 0.447291883 0.265855387
## Llama 2 7b 0.33293488 0.446337734 0.217477060
## Llama 3.1 70b Instruct 0.86085406 0.443552829 0.898077093
## Llama 3.1 70b 1.00000000 0.409176453 0.840792421
## Llama 3.1 8b Instruct 0.40917645 1.000000000 0.533485613
## Llama 3 70b Instruct 0.84079242 0.533485613 1.000000000
## Llama 3 70b 0.97126692 0.459075279 0.864068518
## Llama 3 8b Instruct -0.24733703 -0.081363180 -0.011820515
## Llama 3 8b -0.06978592 -0.038426119 -0.092270527
## Mixtral 8x7b Instruct 0.50063083 0.597191969 0.464313912
## Mixtral 8x7b 0.55460700 0.591095640 0.465740726
## Olmo 2 32b Dpo 0.66508671 0.445455850 0.591764114
## Olmo 2 32b Instruct 0.65722966 0.449593638 0.592803013
## Olmo 2 32b Sft 0.64892502 0.458140403 0.562273867
## Olmo 2 32b 0.69318633 0.394052614 0.586378344
## Olmo 2 1b Dpo 0.10127133 -0.035253076 0.059095088
## Olmo 2 1b Instruct 0.05795927 -0.090043196 0.010082509
## Olmo 2 1b Sft 0.15849691 0.058061912 0.137231782
## Olmo 2 1b -0.02456292 0.096617796 0.064595978
## Olmo 2 13b Dpo 0.55595704 0.611011045 0.663213762
## Olmo 2 13b Instruct 0.56424461 0.610732795 0.671068012
## Olmo 2 13b Sft 0.51046125 0.602627602 0.605835448
## Olmo 2 13b 0.50402418 0.618961950 0.655055928
## Olmo 2 7b Dpo 0.46216828 0.555795209 0.504154097
## Olmo 2 7b Instruct 0.46460200 0.566096612 0.514499608
## Olmo 2 7b Sft 0.42872330 0.526436884 0.486621490
## Olmo 2 7b 0.30434313 0.513587335 0.457911680
## Pythia 1.4b 0.21878567 0.253566395 0.282880257
## Pythia 12b 0.11642442 0.371431276 0.234235450
## Pythia 14m -0.04468846 -0.202569178 -0.188949809
## Pythia 160m -0.14045004 -0.105894152 -0.277848796
## Pythia 1b 0.38172348 0.276656570 0.344366913
## Pythia 2.8b 0.03742366 0.352128348 0.171199127
## Pythia 410m 0.16053774 -0.005627089 0.106752072
## Pythia 6.9b 0.12996782 0.173655392 0.171293278
## Pythia 70m -0.09997244 0.039877150 -0.159879127
## Qwen 2.5 0.5b Instruct 0.13186218 0.035228854 0.060928784
## Qwen 2.5 0.5b 0.07041248 -0.019750613 0.006903611
## Qwen 2.5 1.5 Instruct 0.17031491 -0.005103341 -0.028614507
## Qwen 2.5 1.5b 0.24506094 0.048317929 0.060067359
## Qwen 2.5 14b Instruct 0.54001024 0.417780336 0.514909380
## Qwen 2.5 14b 0.64740850 0.381642020 0.580862418
## Qwen 2.5 32b Instruct 0.34030204 0.267850469 0.458521399
## Qwen 2.5 32b 0.47654577 0.275795725 0.585527870
## Qwen 2.5 3b Instruct 0.02339103 0.059024927 0.054169273
## Qwen 2.5 3b 0.15178875 0.304490251 0.204810882
## Qwen 2.5 72b Instruct 0.38084582 0.314229204 0.563641839
## Qwen 2.5 7b Instruct 0.51658888 0.459818607 0.541649692
## Qwen 2.5 7b 0.35540772 0.460528641 0.454414139
## Human 0.51723693 0.307820396 0.499488089
## Llama 3 70b Llama 3 8b Instruct Llama 3 8b
## Gemma 2 2b Instruct -0.46717965 0.376631659 0.204659447
## Gemma 2 2b -0.34211676 0.030717158 -0.270471477
## Llama 2 13b Instruct 0.44053254 -0.363834021 -0.269952314
## Llama 2 13b 0.38196037 -0.415213609 -0.138375140
## Llama 2 70b Instruct 0.61558193 -0.400962048 -0.257178909
## Llama 2 70b 0.49879545 -0.450317375 -0.041773155
## Llama 2 7b Instruct 0.38269300 -0.364370723 -0.188857045
## Llama 2 7b 0.36271528 -0.428681738 -0.112609341
## Llama 3.1 70b Instruct 0.85468575 -0.050807920 -0.109723268
## Llama 3.1 70b 0.97126692 -0.247337035 -0.069785917
## Llama 3.1 8b Instruct 0.45907528 -0.081363180 -0.038426119
## Llama 3 70b Instruct 0.86406852 -0.011820515 -0.092270527
## Llama 3 70b 1.00000000 -0.229207916 -0.118250812
## Llama 3 8b Instruct -0.22920792 1.000000000 0.446113729
## Llama 3 8b -0.11825081 0.446113729 1.000000000
## Mixtral 8x7b Instruct 0.54900635 -0.189348042 -0.077548002
## Mixtral 8x7b 0.60496049 -0.296550637 -0.102458544
## Olmo 2 32b Dpo 0.66875168 -0.034406162 0.195828288
## Olmo 2 32b Instruct 0.66184913 -0.023335519 0.186966220
## Olmo 2 32b Sft 0.65094053 -0.077149769 0.204129428
## Olmo 2 32b 0.69393029 -0.132513410 0.169344385
## Olmo 2 1b Dpo 0.14498167 -0.082454367 -0.086866355
## Olmo 2 1b Instruct 0.09543385 -0.079699016 -0.051810206
## Olmo 2 1b Sft 0.23158933 -0.042260405 -0.158248176
## Olmo 2 1b -0.03333591 0.178471841 0.205377198
## Olmo 2 13b Dpo 0.56966361 -0.113204479 -0.065440881
## Olmo 2 13b Instruct 0.57663101 -0.112177060 -0.058593468
## Olmo 2 13b Sft 0.53003697 -0.164532535 -0.097869405
## Olmo 2 13b 0.53462676 -0.015505101 -0.053347731
## Olmo 2 7b Dpo 0.47518940 -0.123576188 -0.020576010
## Olmo 2 7b Instruct 0.47846935 -0.122455782 -0.032843156
## Olmo 2 7b Sft 0.44681253 -0.090268168 -0.014475384
## Olmo 2 7b 0.35992463 0.257165586 0.112535184
## Pythia 1.4b 0.19761426 0.135136540 0.219950695
## Pythia 12b 0.13453350 -0.091446109 -0.220672610
## Pythia 14m -0.08652236 -0.122037085 0.425282893
## Pythia 160m -0.18121896 -0.030689773 0.197053687
## Pythia 1b 0.41072493 -0.008486793 0.073604728
## Pythia 2.8b 0.09470626 -0.038601716 -0.049826328
## Pythia 410m 0.17150769 -0.261202466 -0.421282869
## Pythia 6.9b 0.14967203 0.075760424 0.017770155
## Pythia 70m -0.17004700 -0.052836303 0.105603959
## Qwen 2.5 0.5b Instruct 0.11354401 -0.020485786 -0.188110799
## Qwen 2.5 0.5b 0.06035484 0.061590546 0.001420336
## Qwen 2.5 1.5 Instruct 0.14947625 -0.197270451 -0.100168526
## Qwen 2.5 1.5b 0.22624467 -0.204943249 -0.164819152
## Qwen 2.5 14b Instruct 0.55482549 0.126668751 0.244651345
## Qwen 2.5 14b 0.65792841 0.013538822 0.174680264
## Qwen 2.5 32b Instruct 0.36919512 0.001228225 -0.109201760
## Qwen 2.5 32b 0.49356882 -0.122647240 -0.182735101
## Qwen 2.5 3b Instruct 0.04143617 0.246351206 -0.098105445
## Qwen 2.5 3b 0.19631159 0.219186823 -0.112908417
## Qwen 2.5 72b Instruct 0.36419968 0.134932398 0.053596406
## Qwen 2.5 7b Instruct 0.54101562 -0.168362789 -0.006157379
## Qwen 2.5 7b 0.39129274 -0.133100532 -0.005530676
## Human 0.48424530 0.028148418 0.052040977
## Mixtral 8x7b Instruct Mixtral 8x7b Olmo 2 32b Dpo
## Gemma 2 2b Instruct -0.45606603 -0.514902287 -0.26873199
## Gemma 2 2b -0.28461451 -0.309120341 -0.32769037
## Llama 2 13b Instruct 0.62652149 0.697092113 0.31291487
## Llama 2 13b 0.62244564 0.739906704 0.29479476
## Llama 2 70b Instruct 0.59956069 0.723789180 0.44682547
## Llama 2 70b 0.58912332 0.735760866 0.37665105
## Llama 2 7b Instruct 0.56177709 0.678456336 0.26985902
## Llama 2 7b 0.59063184 0.716744420 0.28371613
## Llama 3.1 70b Instruct 0.43324826 0.459139398 0.58956655
## Llama 3.1 70b 0.50063083 0.554607002 0.66508671
## Llama 3.1 8b Instruct 0.59719197 0.591095640 0.44545585
## Llama 3 70b Instruct 0.46431391 0.465740726 0.59176411
## Llama 3 70b 0.54900635 0.604960489 0.66875168
## Llama 3 8b Instruct -0.18934804 -0.296550637 -0.03440616
## Llama 3 8b -0.07754800 -0.102458544 0.19582829
## Mixtral 8x7b Instruct 1.00000000 0.942716981 0.43279527
## Mixtral 8x7b 0.94271698 1.000000000 0.48340113
## Olmo 2 32b Dpo 0.43279527 0.483401128 1.00000000
## Olmo 2 32b Instruct 0.43807558 0.482558989 0.99842363
## Olmo 2 32b Sft 0.38860088 0.460821935 0.97310982
## Olmo 2 32b 0.39003422 0.464677462 0.92893850
## Olmo 2 1b Dpo 0.05577211 0.083124551 0.18997057
## Olmo 2 1b Instruct 0.01126859 0.042769278 0.16359673
## Olmo 2 1b Sft 0.17024035 0.178160003 0.23413146
## Olmo 2 1b -0.10533078 -0.036090736 0.02686042
## Olmo 2 13b Dpo 0.50175132 0.538648284 0.49538442
## Olmo 2 13b Instruct 0.49728755 0.536659579 0.50758550
## Olmo 2 13b Sft 0.52742353 0.572014291 0.47589628
## Olmo 2 13b 0.46846067 0.485686028 0.43637212
## Olmo 2 7b Dpo 0.30017668 0.319809011 0.42913757
## Olmo 2 7b Instruct 0.30648467 0.323839765 0.42952589
## Olmo 2 7b Sft 0.24464144 0.267724273 0.39344614
## Olmo 2 7b 0.29409683 0.270431250 0.30010754
## Pythia 1.4b 0.09564842 0.151999551 0.03006352
## Pythia 12b 0.11728209 0.183177956 -0.01459159
## Pythia 14m 0.07741670 0.139140355 0.04440013
## Pythia 160m -0.00473788 0.003189803 -0.04495257
## Pythia 1b 0.28193277 0.343824629 0.33934434
## Pythia 2.8b 0.11318734 0.135495370 0.01432867
## Pythia 410m 0.17000364 0.165957107 -0.03593991
## Pythia 6.9b 0.06472750 0.046994532 0.09203062
## Pythia 70m -0.14628581 -0.119071195 0.08511552
## Qwen 2.5 0.5b Instruct -0.11437566 -0.112317444 0.05049015
## Qwen 2.5 0.5b -0.08760174 -0.078910118 0.01477185
## Qwen 2.5 1.5 Instruct 0.11920858 0.227064651 0.22382226
## Qwen 2.5 1.5b 0.14141271 0.262492354 0.20803906
## Qwen 2.5 14b Instruct 0.21721231 0.207378799 0.55168534
## Qwen 2.5 14b 0.26302285 0.313163455 0.55183577
## Qwen 2.5 32b Instruct 0.12770873 0.114713763 0.28345055
## Qwen 2.5 32b 0.11828924 0.108881287 0.33276187
## Qwen 2.5 3b Instruct 0.01595507 -0.006845552 0.15932196
## Qwen 2.5 3b 0.28649586 0.271176411 0.25710327
## Qwen 2.5 72b Instruct 0.25136701 0.145834337 0.26202766
## Qwen 2.5 7b Instruct 0.22832156 0.259136665 0.40119218
## Qwen 2.5 7b 0.10869269 0.154419588 0.26133639
## Human 0.33944765 0.318033699 0.49268535
## Olmo 2 32b Instruct Olmo 2 32b Sft Olmo 2 32b
## Gemma 2 2b Instruct -0.265877635 -0.22203150 -0.25961750
## Gemma 2 2b -0.324959759 -0.28974565 -0.29506063
## Llama 2 13b Instruct 0.308630576 0.30934691 0.34551229
## Llama 2 13b 0.286943059 0.32050269 0.34073750
## Llama 2 70b Instruct 0.442706864 0.44709231 0.46312550
## Llama 2 70b 0.364903822 0.39823696 0.43843985
## Llama 2 7b Instruct 0.262343500 0.27332307 0.29907047
## Llama 2 7b 0.276209024 0.30718059 0.32391648
## Llama 3.1 70b Instruct 0.591725678 0.54846014 0.55625015
## Llama 3.1 70b 0.657229663 0.64892502 0.69318633
## Llama 3.1 8b Instruct 0.449593638 0.45814040 0.39405261
## Llama 3 70b Instruct 0.592803013 0.56227387 0.58637834
## Llama 3 70b 0.661849129 0.65094053 0.69393029
## Llama 3 8b Instruct -0.023335519 -0.07714977 -0.13251341
## Llama 3 8b 0.186966220 0.20412943 0.16934438
## Mixtral 8x7b Instruct 0.438075580 0.38860088 0.39003422
## Mixtral 8x7b 0.482558989 0.46082194 0.46467746
## Olmo 2 32b Dpo 0.998423631 0.97310982 0.92893850
## Olmo 2 32b Instruct 1.000000000 0.96676238 0.91578491
## Olmo 2 32b Sft 0.966762376 1.00000000 0.95127471
## Olmo 2 32b 0.915784912 0.95127471 1.00000000
## Olmo 2 1b Dpo 0.188431451 0.18635608 0.22751783
## Olmo 2 1b Instruct 0.160194241 0.16548010 0.20886472
## Olmo 2 1b Sft 0.236045166 0.22176187 0.25811954
## Olmo 2 1b 0.027845343 0.09550461 0.05242931
## Olmo 2 13b Dpo 0.485727352 0.53104746 0.53630203
## Olmo 2 13b Instruct 0.498043853 0.54359023 0.54588190
## Olmo 2 13b Sft 0.465940515 0.50830469 0.51433926
## Olmo 2 13b 0.427827584 0.46404018 0.47430018
## Olmo 2 7b Dpo 0.420789033 0.50200256 0.52532685
## Olmo 2 7b Instruct 0.421769247 0.49971169 0.52178025
## Olmo 2 7b Sft 0.386642263 0.46584849 0.48355855
## Olmo 2 7b 0.300632612 0.33587857 0.35207117
## Pythia 1.4b 0.022020759 0.04307584 0.04618491
## Pythia 12b -0.011477420 0.02926448 0.03264969
## Pythia 14m 0.033887343 0.05204865 0.06067670
## Pythia 160m -0.052136591 0.03675959 0.02264439
## Pythia 1b 0.340253931 0.35764612 0.34508557
## Pythia 2.8b 0.010657764 0.06578982 0.09181197
## Pythia 410m -0.036242930 -0.01368577 0.03966343
## Pythia 6.9b 0.092584295 0.09195912 0.15927584
## Pythia 70m 0.077889772 0.16750206 0.16728886
## Qwen 2.5 0.5b Instruct 0.041378227 0.10582455 0.18513817
## Qwen 2.5 0.5b 0.006072844 0.09283052 0.14909186
## Qwen 2.5 1.5 Instruct 0.223301563 0.23013883 0.24397477
## Qwen 2.5 1.5b 0.209896151 0.21707263 0.23338514
## Qwen 2.5 14b Instruct 0.546933717 0.57874043 0.55699388
## Qwen 2.5 14b 0.539831586 0.59950363 0.60565887
## Qwen 2.5 32b Instruct 0.285411703 0.29418103 0.32739804
## Qwen 2.5 32b 0.330272954 0.33453898 0.37418949
## Qwen 2.5 3b Instruct 0.163240947 0.18441628 0.19573253
## Qwen 2.5 3b 0.259058756 0.25456494 0.27366834
## Qwen 2.5 72b Instruct 0.266573197 0.24982204 0.25607232
## Qwen 2.5 7b Instruct 0.402293738 0.43206625 0.41226966
## Qwen 2.5 7b 0.263540114 0.31647610 0.29544058
## Human 0.486532058 0.47361359 0.49408747
## Olmo 2 1b Dpo Olmo 2 1b Instruct Olmo 2 1b Sft
## Gemma 2 2b Instruct -0.3668719930 -0.310201372 -0.429979033
## Gemma 2 2b -0.1744724378 -0.140375157 -0.250924309
## Llama 2 13b Instruct 0.1051888144 0.073647917 0.159629937
## Llama 2 13b 0.0492988142 0.039881865 0.070487137
## Llama 2 70b Instruct 0.1824556055 0.135316704 0.241859390
## Llama 2 70b 0.1435890148 0.133570917 0.156725305
## Llama 2 7b Instruct 0.0573188501 0.041950545 0.077973331
## Llama 2 7b 0.0873768770 0.086024836 0.098420313
## Llama 3.1 70b Instruct 0.0403496366 -0.004349600 0.122816361
## Llama 3.1 70b 0.1012713261 0.057959273 0.158496909
## Llama 3.1 8b Instruct -0.0352530765 -0.090043196 0.058061912
## Llama 3 70b Instruct 0.0590950884 0.010082509 0.137231782
## Llama 3 70b 0.1449816728 0.095433850 0.231589331
## Llama 3 8b Instruct -0.0824543668 -0.079699016 -0.042260405
## Llama 3 8b -0.0868663553 -0.051810206 -0.158248176
## Mixtral 8x7b Instruct 0.0557721122 0.011268593 0.170240351
## Mixtral 8x7b 0.0831245508 0.042769278 0.178160003
## Olmo 2 32b Dpo 0.1899705736 0.163596733 0.234131465
## Olmo 2 32b Instruct 0.1884314513 0.160194241 0.236045166
## Olmo 2 32b Sft 0.1863560824 0.165480097 0.221761869
## Olmo 2 32b 0.2275178302 0.208864718 0.258119542
## Olmo 2 1b Dpo 1.0000000000 0.990304776 0.930559083
## Olmo 2 1b Instruct 0.9903047756 1.000000000 0.903794088
## Olmo 2 1b Sft 0.9305590830 0.903794088 1.000000000
## Olmo 2 1b -0.1205451619 -0.098906337 -0.111310867
## Olmo 2 13b Dpo -0.0161128886 -0.047880134 0.006284633
## Olmo 2 13b Instruct -0.0175222180 -0.049179564 0.005036481
## Olmo 2 13b Sft 0.0043886244 -0.028054281 0.032102806
## Olmo 2 13b 0.0162093748 -0.022045443 0.056499619
## Olmo 2 7b Dpo -0.0048150303 -0.034164957 0.071672642
## Olmo 2 7b Instruct -0.0137691676 -0.045054745 0.062644862
## Olmo 2 7b Sft 0.0134511363 -0.011883333 0.104903117
## Olmo 2 7b 0.0005876432 -0.041391561 0.099566255
## Pythia 1.4b -0.1347737416 -0.124267731 -0.141887165
## Pythia 12b -0.0572204460 -0.073681650 -0.065479952
## Pythia 14m -0.1887870942 -0.160030904 -0.201939198
## Pythia 160m -0.1230380399 -0.086767891 -0.164196100
## Pythia 1b 0.3559699277 0.312690457 0.421374527
## Pythia 2.8b 0.2215166814 0.217493550 0.228853205
## Pythia 410m 0.4216237690 0.398387109 0.370064544
## Pythia 6.9b 0.1664297714 0.146954385 0.238639449
## Pythia 70m -0.2256344037 -0.211977162 -0.275763816
## Qwen 2.5 0.5b Instruct 0.0619853527 0.051003262 0.096267196
## Qwen 2.5 0.5b 0.1189521841 0.119688341 0.139796770
## Qwen 2.5 1.5 Instruct 0.2826837202 0.289992344 0.249415485
## Qwen 2.5 1.5b 0.2291535327 0.222421662 0.210533426
## Qwen 2.5 14b Instruct 0.0070861754 -0.017389356 0.071088587
## Qwen 2.5 14b 0.0206682048 0.007877412 0.087526054
## Qwen 2.5 32b Instruct -0.0300432226 -0.072499526 0.035085114
## Qwen 2.5 32b 0.0250908581 -0.021668942 0.112688898
## Qwen 2.5 3b Instruct 0.1798014326 0.146527580 0.262349855
## Qwen 2.5 3b 0.2166372868 0.164310684 0.374894055
## Qwen 2.5 72b Instruct -0.1436185242 -0.165211161 -0.117443789
## Qwen 2.5 7b Instruct 0.1593747618 0.128976191 0.186568705
## Qwen 2.5 7b 0.0901529194 0.069361322 0.105574230
## Human -0.0294047682 -0.042456790 -0.037876419
## Olmo 2 1b Olmo 2 13b Dpo Olmo 2 13b Instruct
## Gemma 2 2b Instruct 0.334363799 -0.0557962311 -0.053642530
## Gemma 2 2b 0.028223587 0.0005462096 -0.003223577
## Llama 2 13b Instruct -0.093465625 0.5168539777 0.512386340
## Llama 2 13b 0.080015299 0.4629899638 0.459307362
## Llama 2 70b Instruct 0.073944474 0.5173617337 0.523234890
## Llama 2 70b 0.090861006 0.4140093953 0.416929224
## Llama 2 7b Instruct 0.068027782 0.4334077398 0.429894259
## Llama 2 7b 0.106162239 0.4104798326 0.406384254
## Llama 3.1 70b Instruct 0.096770860 0.5610407730 0.570516638
## Llama 3.1 70b -0.024562922 0.5559570395 0.564244613
## Llama 3.1 8b Instruct 0.096617796 0.6110110455 0.610732795
## Llama 3 70b Instruct 0.064595978 0.6632137620 0.671068012
## Llama 3 70b -0.033335908 0.5696636125 0.576631008
## Llama 3 8b Instruct 0.178471841 -0.1132044787 -0.112177060
## Llama 3 8b 0.205377198 -0.0654408811 -0.058593468
## Mixtral 8x7b Instruct -0.105330779 0.5017513197 0.497287554
## Mixtral 8x7b -0.036090736 0.5386482841 0.536659579
## Olmo 2 32b Dpo 0.026860417 0.4953844181 0.507585501
## Olmo 2 32b Instruct 0.027845343 0.4857273516 0.498043853
## Olmo 2 32b Sft 0.095504608 0.5310474572 0.543590232
## Olmo 2 32b 0.052429307 0.5363020337 0.545881904
## Olmo 2 1b Dpo -0.120545162 -0.0161128886 -0.017522218
## Olmo 2 1b Instruct -0.098906337 -0.0478801342 -0.049179564
## Olmo 2 1b Sft -0.111310867 0.0062846332 0.005036481
## Olmo 2 1b 1.000000000 0.1031944866 0.117557869
## Olmo 2 13b Dpo 0.103194487 1.0000000000 0.998914719
## Olmo 2 13b Instruct 0.117557869 0.9989147193 1.000000000
## Olmo 2 13b Sft 0.070845916 0.9797863093 0.974952843
## Olmo 2 13b 0.057065511 0.9461344116 0.941044623
## Olmo 2 7b Dpo 0.119526664 0.6265647468 0.631132959
## Olmo 2 7b Instruct 0.112669504 0.6328774009 0.637337044
## Olmo 2 7b Sft 0.190702353 0.5977515079 0.604115254
## Olmo 2 7b 0.189332442 0.5296084690 0.529616174
## Pythia 1.4b 0.146354704 0.2655715266 0.265261076
## Pythia 12b 0.222779580 0.4151317749 0.414389686
## Pythia 14m -0.001917153 -0.1196895128 -0.115524337
## Pythia 160m 0.120059530 0.0398685511 0.034760596
## Pythia 1b 0.040890215 0.1057222408 0.113218839
## Pythia 2.8b 0.218756160 0.1701853192 0.171326300
## Pythia 410m -0.384083970 0.1144700882 0.103617608
## Pythia 6.9b 0.153344714 0.0032498434 0.004192458
## Pythia 70m 0.112500652 0.1100751350 0.110481207
## Qwen 2.5 0.5b Instruct 0.052144108 0.1007109701 0.097050893
## Qwen 2.5 0.5b 0.066047125 0.0014700857 -0.001451533
## Qwen 2.5 1.5 Instruct -0.029098896 0.1789545750 0.181110574
## Qwen 2.5 1.5b 0.024424368 0.2165970121 0.217891888
## Qwen 2.5 14b Instruct 0.087913540 0.3171281374 0.322909278
## Qwen 2.5 14b 0.227187565 0.4431815427 0.453684768
## Qwen 2.5 32b Instruct 0.106164288 0.2478215586 0.257688254
## Qwen 2.5 32b 0.038593774 0.3147997025 0.324572046
## Qwen 2.5 3b Instruct 0.022651858 -0.0053558096 -0.003185889
## Qwen 2.5 3b -0.027422935 0.1455595321 0.147052815
## Qwen 2.5 72b Instruct 0.017673172 0.4267350774 0.427929209
## Qwen 2.5 7b Instruct 0.066949770 0.3914112388 0.397348385
## Qwen 2.5 7b 0.148268886 0.3437470969 0.346546033
## Human -0.049929726 0.5083238940 0.508172353
## Olmo 2 13b Sft Olmo 2 13b Olmo 2 7b Dpo
## Gemma 2 2b Instruct -0.106839894 -0.057371452 0.01216161
## Gemma 2 2b -0.022504672 -0.020157896 0.02950962
## Llama 2 13b Instruct 0.543185459 0.507096428 0.42999293
## Llama 2 13b 0.506657649 0.429371098 0.36369714
## Llama 2 70b Instruct 0.535459054 0.483474925 0.37884347
## Llama 2 70b 0.443700349 0.362347777 0.30852715
## Llama 2 7b Instruct 0.484442279 0.417944041 0.32596276
## Llama 2 7b 0.463289995 0.376403961 0.31488415
## Llama 3.1 70b Instruct 0.509944831 0.509178125 0.36135596
## Llama 3.1 70b 0.510461251 0.504024178 0.46216828
## Llama 3.1 8b Instruct 0.602627602 0.618961950 0.55579521
## Llama 3 70b Instruct 0.605835448 0.655055928 0.50415410
## Llama 3 70b 0.530036973 0.534626764 0.47518940
## Llama 3 8b Instruct -0.164532535 -0.015505101 -0.12357619
## Llama 3 8b -0.097869405 -0.053347731 -0.02057601
## Mixtral 8x7b Instruct 0.527423535 0.468460671 0.30017668
## Mixtral 8x7b 0.572014291 0.485686028 0.31980901
## Olmo 2 32b Dpo 0.475896278 0.436372120 0.42913757
## Olmo 2 32b Instruct 0.465940515 0.427827584 0.42078903
## Olmo 2 32b Sft 0.508304690 0.464040183 0.50200256
## Olmo 2 32b 0.514339260 0.474300178 0.52532685
## Olmo 2 1b Dpo 0.004388624 0.016209375 -0.00481503
## Olmo 2 1b Instruct -0.028054281 -0.022045443 -0.03416496
## Olmo 2 1b Sft 0.032102806 0.056499619 0.07167264
## Olmo 2 1b 0.070845916 0.057065511 0.11952666
## Olmo 2 13b Dpo 0.979786309 0.946134412 0.62656475
## Olmo 2 13b Instruct 0.974952843 0.941044623 0.63113296
## Olmo 2 13b Sft 1.000000000 0.941665762 0.59733456
## Olmo 2 13b 0.941665762 1.000000000 0.62035384
## Olmo 2 7b Dpo 0.597334558 0.620353840 1.00000000
## Olmo 2 7b Instruct 0.603883176 0.628785443 0.99903213
## Olmo 2 7b Sft 0.570609998 0.594466701 0.97304562
## Olmo 2 7b 0.505039786 0.632616768 0.71398207
## Pythia 1.4b 0.217111217 0.276951333 0.28630209
## Pythia 12b 0.394179955 0.441688763 0.47741483
## Pythia 14m -0.092422134 -0.172730554 -0.10135782
## Pythia 160m 0.062367212 -0.006062482 -0.03220031
## Pythia 1b 0.085608135 0.136232735 0.13369550
## Pythia 2.8b 0.129061630 0.196461631 0.29208159
## Pythia 410m 0.118852081 0.137740115 0.08061162
## Pythia 6.9b -0.028836128 0.065062712 0.31735533
## Pythia 70m 0.108660027 0.087402412 0.19671280
## Qwen 2.5 0.5b Instruct 0.100604779 0.165907277 0.26079462
## Qwen 2.5 0.5b -0.011096335 0.033207760 0.23342037
## Qwen 2.5 1.5 Instruct 0.204805400 0.152631531 0.17863468
## Qwen 2.5 1.5b 0.235540322 0.183360282 0.13095480
## Qwen 2.5 14b Instruct 0.258818474 0.345771998 0.49930574
## Qwen 2.5 14b 0.379667331 0.435602337 0.54424038
## Qwen 2.5 32b Instruct 0.199356901 0.285343401 0.42925766
## Qwen 2.5 32b 0.267380226 0.332179885 0.44812931
## Qwen 2.5 3b Instruct 0.007940175 0.045082532 0.22365222
## Qwen 2.5 3b 0.169311726 0.201722726 0.34365117
## Qwen 2.5 72b Instruct 0.379499726 0.419267323 0.33971835
## Qwen 2.5 7b Instruct 0.361696892 0.439806610 0.44424877
## Qwen 2.5 7b 0.309776666 0.388440172 0.37634211
## Human 0.486740785 0.489965655 0.38164894
## Olmo 2 7b Instruct Olmo 2 7b Sft Olmo 2 7b
## Gemma 2 2b Instruct 0.01240507 0.03787573 -0.0316540868
## Gemma 2 2b 0.04340489 -0.01156918 -0.1675830716
## Llama 2 13b Instruct 0.44161895 0.38095705 0.3627548954
## Llama 2 13b 0.36812844 0.34165108 0.3255947787
## Llama 2 70b Instruct 0.38647439 0.35708366 0.3684886798
## Llama 2 70b 0.30590194 0.28177885 0.2701557493
## Llama 2 7b Instruct 0.33446204 0.31110620 0.3259980128
## Llama 2 7b 0.31755264 0.30245569 0.2702910813
## Llama 3.1 70b Instruct 0.37069583 0.35063386 0.2966559774
## Llama 3.1 70b 0.46460200 0.42872330 0.3043431311
## Llama 3.1 8b Instruct 0.56609661 0.52643688 0.5135873351
## Llama 3 70b Instruct 0.51449961 0.48662149 0.4579116802
## Llama 3 70b 0.47846935 0.44681253 0.3599246268
## Llama 3 8b Instruct -0.12245578 -0.09026817 0.2571655863
## Llama 3 8b -0.03284316 -0.01447538 0.1125351839
## Mixtral 8x7b Instruct 0.30648467 0.24464144 0.2940968350
## Mixtral 8x7b 0.32383976 0.26772427 0.2704312497
## Olmo 2 32b Dpo 0.42952589 0.39344614 0.3001075365
## Olmo 2 32b Instruct 0.42176925 0.38664226 0.3006326124
## Olmo 2 32b Sft 0.49971169 0.46584849 0.3358785656
## Olmo 2 32b 0.52178025 0.48355855 0.3520711679
## Olmo 2 1b Dpo -0.01376917 0.01345114 0.0005876432
## Olmo 2 1b Instruct -0.04505474 -0.01188333 -0.0413915608
## Olmo 2 1b Sft 0.06264486 0.10490312 0.0995662550
## Olmo 2 1b 0.11266950 0.19070235 0.1893324421
## Olmo 2 13b Dpo 0.63287740 0.59775151 0.5296084690
## Olmo 2 13b Instruct 0.63733704 0.60411525 0.5296161735
## Olmo 2 13b Sft 0.60388318 0.57061000 0.5050397864
## Olmo 2 13b 0.62878544 0.59446670 0.6326167680
## Olmo 2 7b Dpo 0.99903213 0.97304562 0.7139820706
## Olmo 2 7b Instruct 1.00000000 0.96799510 0.7171743096
## Olmo 2 7b Sft 0.96799510 1.00000000 0.7221305980
## Olmo 2 7b 0.71717431 0.72213060 1.0000000000
## Pythia 1.4b 0.28765418 0.30082703 0.2530041631
## Pythia 12b 0.48846032 0.48356270 0.5071267285
## Pythia 14m -0.10785845 -0.14759881 -0.2433842066
## Pythia 160m -0.04417796 -0.06462760 -0.0977332093
## Pythia 1b 0.13507121 0.10620891 0.2667100660
## Pythia 2.8b 0.29210058 0.30266183 0.2787189857
## Pythia 410m 0.08118128 0.02742072 0.0236920300
## Pythia 6.9b 0.31896405 0.34243780 0.3959930998
## Pythia 70m 0.19730627 0.15421563 0.0667561238
## Qwen 2.5 0.5b Instruct 0.25331789 0.26490363 0.3334774669
## Qwen 2.5 0.5b 0.22241616 0.22344018 0.2834512935
## Qwen 2.5 1.5 Instruct 0.17471874 0.15467891 0.0166640079
## Qwen 2.5 1.5b 0.12881585 0.10995943 0.0214309431
## Qwen 2.5 14b Instruct 0.49211743 0.46833045 0.3948476316
## Qwen 2.5 14b 0.53255144 0.54781206 0.4007199987
## Qwen 2.5 32b Instruct 0.43526338 0.40311445 0.3984433953
## Qwen 2.5 32b 0.45464078 0.43360342 0.3086692579
## Qwen 2.5 3b Instruct 0.22371647 0.22769639 0.2057211102
## Qwen 2.5 3b 0.34522791 0.33440911 0.3906292594
## Qwen 2.5 72b Instruct 0.35374799 0.30318732 0.2770573341
## Qwen 2.5 7b Instruct 0.44733848 0.42219988 0.3935283655
## Qwen 2.5 7b 0.37761616 0.37361306 0.3312800530
## Human 0.39024048 0.31516444 0.2684078998
## Pythia 1.4b Pythia 12b Pythia 14m Pythia 160m
## Gemma 2 2b Instruct 0.009603831 0.04490751 -0.122810764 0.270712660
## Gemma 2 2b 0.024179412 0.21798214 -0.087940208 0.038639682
## Llama 2 13b Instruct 0.268290222 0.46597188 0.005675609 -0.182986484
## Llama 2 13b 0.290153425 0.44959787 0.094687910 -0.148554379
## Llama 2 70b Instruct 0.213301303 0.44915881 -0.063687856 -0.288434497
## Llama 2 70b 0.207333777 0.35915319 0.227873924 -0.130818144
## Llama 2 7b Instruct 0.332839311 0.52114393 0.073199503 -0.160759380
## Llama 2 7b 0.272648132 0.45821914 0.151960716 -0.141312090
## Llama 3.1 70b Instruct 0.184671249 0.18581031 -0.141310142 -0.232494334
## Llama 3.1 70b 0.218785666 0.11642442 -0.044688464 -0.140450036
## Llama 3.1 8b Instruct 0.253566395 0.37143128 -0.202569178 -0.105894152
## Llama 3 70b Instruct 0.282880257 0.23423545 -0.188949809 -0.277848796
## Llama 3 70b 0.197614260 0.13453350 -0.086522360 -0.181218959
## Llama 3 8b Instruct 0.135136540 -0.09144611 -0.122037085 -0.030689773
## Llama 3 8b 0.219950695 -0.22067261 0.425282893 0.197053687
## Mixtral 8x7b Instruct 0.095648416 0.11728209 0.077416705 -0.004737880
## Mixtral 8x7b 0.151999551 0.18317796 0.139140355 0.003189803
## Olmo 2 32b Dpo 0.030063521 -0.01459159 0.044400126 -0.044952566
## Olmo 2 32b Instruct 0.022020759 -0.01147742 0.033887343 -0.052136591
## Olmo 2 32b Sft 0.043075840 0.02926448 0.052048651 0.036759591
## Olmo 2 32b 0.046184914 0.03264969 0.060676705 0.022644389
## Olmo 2 1b Dpo -0.134773742 -0.05722045 -0.188787094 -0.123038040
## Olmo 2 1b Instruct -0.124267731 -0.07368165 -0.160030904 -0.086767891
## Olmo 2 1b Sft -0.141887165 -0.06547995 -0.201939198 -0.164196100
## Olmo 2 1b 0.146354704 0.22277958 -0.001917153 0.120059530
## Olmo 2 13b Dpo 0.265571527 0.41513177 -0.119689513 0.039868551
## Olmo 2 13b Instruct 0.265261076 0.41438969 -0.115524337 0.034760596
## Olmo 2 13b Sft 0.217111217 0.39417995 -0.092422134 0.062367212
## Olmo 2 13b 0.276951333 0.44168876 -0.172730554 -0.006062482
## Olmo 2 7b Dpo 0.286302094 0.47741483 -0.101357823 -0.032200308
## Olmo 2 7b Instruct 0.287654185 0.48846032 -0.107858449 -0.044177962
## Olmo 2 7b Sft 0.300827026 0.48356270 -0.147598811 -0.064627599
## Olmo 2 7b 0.253004163 0.50712673 -0.243384207 -0.097733209
## Pythia 1.4b 1.000000000 0.44038391 0.125798687 -0.087970296
## Pythia 12b 0.440383907 1.00000000 -0.172419706 -0.136651896
## Pythia 14m 0.125798687 -0.17241971 1.000000000 0.246905955
## Pythia 160m -0.087970296 -0.13665190 0.246905955 1.000000000
## Pythia 1b 0.146925766 0.22405541 0.048746691 -0.037728231
## Pythia 2.8b 0.278815398 0.52605933 -0.296317623 -0.295013355
## Pythia 410m 0.024466820 0.23512480 -0.207959382 0.139834211
## Pythia 6.9b 0.239425471 0.37247587 -0.264887355 -0.421297464
## Pythia 70m 0.070019010 0.10578718 0.223135329 0.497411846
## Qwen 2.5 0.5b Instruct -0.078472972 0.16181069 -0.303909377 0.223357477
## Qwen 2.5 0.5b -0.014560470 0.18303761 -0.173731066 0.271353388
## Qwen 2.5 1.5 Instruct 0.097867094 0.20265159 0.049924767 0.169582139
## Qwen 2.5 1.5b 0.101356842 0.25989029 0.030639761 0.127688446
## Qwen 2.5 14b Instruct 0.225932065 0.03083183 -0.130379289 0.090245569
## Qwen 2.5 14b 0.348172886 0.17052312 -0.042119554 0.008123214
## Qwen 2.5 32b Instruct 0.013803214 0.13930967 -0.205360317 0.038519162
## Qwen 2.5 32b 0.051124713 0.10937750 -0.138445772 -0.187426563
## Qwen 2.5 3b Instruct -0.196126790 -0.03884673 -0.059521536 0.108877714
## Qwen 2.5 3b -0.062993052 0.05393751 -0.006929151 0.012089616
## Qwen 2.5 72b Instruct 0.096137844 0.07077963 -0.191349501 -0.026058025
## Qwen 2.5 7b Instruct 0.158092820 0.20730717 -0.118474847 -0.035566112
## Qwen 2.5 7b 0.117395025 0.20624109 -0.230314239 -0.069084418
## Human 0.077552521 0.07143812 -0.042764449 0.048925527
## Pythia 1b Pythia 2.8b Pythia 410m Pythia 6.9b
## Gemma 2 2b Instruct -0.526743842 0.11051014 -0.395807007 -4.170853e-02
## Gemma 2 2b -0.298645555 0.26051357 0.064701613 -4.944004e-02
## Llama 2 13b Instruct 0.359508222 0.37565349 0.271912692 2.790014e-01
## Llama 2 13b 0.307561976 0.37089701 0.136403077 2.459404e-01
## Llama 2 70b Instruct 0.438075010 0.36034944 0.169809632 2.774241e-01
## Llama 2 70b 0.369026045 0.34556547 0.134458279 2.132556e-01
## Llama 2 7b Instruct 0.304056219 0.29364190 0.160859247 2.505576e-01
## Llama 2 7b 0.279450965 0.35271878 0.126333374 2.366293e-01
## Llama 3.1 70b Instruct 0.359310807 0.10894154 0.077537350 1.205582e-01
## Llama 3.1 70b 0.381723480 0.03742366 0.160537738 1.299678e-01
## Llama 3.1 8b Instruct 0.276656570 0.35212835 -0.005627089 1.736554e-01
## Llama 3 70b Instruct 0.344366913 0.17119913 0.106752072 1.712933e-01
## Llama 3 70b 0.410724927 0.09470626 0.171507687 1.496720e-01
## Llama 3 8b Instruct -0.008486793 -0.03860172 -0.261202466 7.576042e-02
## Llama 3 8b 0.073604728 -0.04982633 -0.421282869 1.777015e-02
## Mixtral 8x7b Instruct 0.281932767 0.11318734 0.170003644 6.472750e-02
## Mixtral 8x7b 0.343824629 0.13549537 0.165957107 4.699453e-02
## Olmo 2 32b Dpo 0.339344345 0.01432867 -0.035939914 9.203062e-02
## Olmo 2 32b Instruct 0.340253931 0.01065776 -0.036242930 9.258429e-02
## Olmo 2 32b Sft 0.357646119 0.06578982 -0.013685770 9.195912e-02
## Olmo 2 32b 0.345085571 0.09181197 0.039663429 1.592758e-01
## Olmo 2 1b Dpo 0.355969928 0.22151668 0.421623769 1.664298e-01
## Olmo 2 1b Instruct 0.312690457 0.21749355 0.398387109 1.469544e-01
## Olmo 2 1b Sft 0.421374527 0.22885320 0.370064544 2.386394e-01
## Olmo 2 1b 0.040890215 0.21875616 -0.384083970 1.533447e-01
## Olmo 2 13b Dpo 0.105722241 0.17018532 0.114470088 3.249843e-03
## Olmo 2 13b Instruct 0.113218839 0.17132630 0.103617608 4.192458e-03
## Olmo 2 13b Sft 0.085608135 0.12906163 0.118852081 -2.883613e-02
## Olmo 2 13b 0.136232735 0.19646163 0.137740115 6.506271e-02
## Olmo 2 7b Dpo 0.133695502 0.29208159 0.080611618 3.173553e-01
## Olmo 2 7b Instruct 0.135071206 0.29210058 0.081181283 3.189641e-01
## Olmo 2 7b Sft 0.106208909 0.30266183 0.027420720 3.424378e-01
## Olmo 2 7b 0.266710066 0.27871899 0.023692030 3.959931e-01
## Pythia 1.4b 0.146925766 0.27881540 0.024466820 2.394255e-01
## Pythia 12b 0.224055411 0.52605933 0.235124798 3.724759e-01
## Pythia 14m 0.048746691 -0.29631762 -0.207959382 -2.648874e-01
## Pythia 160m -0.037728231 -0.29501336 0.139834211 -4.212975e-01
## Pythia 1b 1.000000000 0.31466844 0.408479845 2.765660e-01
## Pythia 2.8b 0.314668441 1.00000000 0.130923504 6.175620e-01
## Pythia 410m 0.408479845 0.13092350 1.000000000 2.376283e-02
## Pythia 6.9b 0.276566012 0.61756198 0.023762835 1.000000e+00
## Pythia 70m -0.156856196 -0.14392443 -0.010271368 -1.803358e-01
## Qwen 2.5 0.5b Instruct 0.159131082 0.06639256 0.279965405 2.258076e-01
## Qwen 2.5 0.5b 0.333429348 0.14254549 0.364932063 2.165243e-01
## Qwen 2.5 1.5 Instruct 0.175297682 0.03476625 0.281315719 -1.174333e-02
## Qwen 2.5 1.5b 0.310767877 0.09777910 0.321535373 -1.490894e-02
## Qwen 2.5 14b Instruct 0.265251740 0.14236706 0.028768751 1.480418e-01
## Qwen 2.5 14b 0.261849086 0.29377271 -0.042767890 3.040191e-01
## Qwen 2.5 32b Instruct 0.038052770 0.06931626 0.013964043 1.822105e-01
## Qwen 2.5 32b 0.011064686 0.06242301 -0.035558688 2.295647e-01
## Qwen 2.5 3b Instruct 0.028574586 -0.06994212 0.105746393 9.972603e-02
## Qwen 2.5 3b 0.237578207 0.02120994 0.141399357 2.111005e-01
## Qwen 2.5 72b Instruct -0.014757472 0.13241416 0.004869379 1.245918e-01
## Qwen 2.5 7b Instruct 0.344988758 0.18651573 0.171168913 5.837450e-02
## Qwen 2.5 7b 0.260466476 0.32445541 0.093188336 1.228564e-01
## Human -0.040319184 -0.03594982 0.039820289 8.817533e-05
## Pythia 70m Qwen 2.5 0.5b Instruct Qwen 2.5 0.5b
## Gemma 2 2b Instruct 0.2942865675 -0.013313842 -0.0418157692
## Gemma 2 2b 0.1237175419 -0.169575439 -0.1046034429
## Llama 2 13b Instruct -0.0062642523 0.092635402 0.0266964466
## Llama 2 13b -0.0370780493 -0.010331259 -0.0388628329
## Llama 2 70b Instruct -0.0817809274 0.084259182 0.0259023944
## Llama 2 70b -0.0647124820 -0.034349290 0.0121153038
## Llama 2 7b Instruct -0.0065813173 0.052594364 0.0016336866
## Llama 2 7b -0.0625011888 -0.069418797 -0.0507075711
## Llama 3.1 70b Instruct -0.1802177952 0.066313337 0.0246857954
## Llama 3.1 70b -0.0999724422 0.131862181 0.0704124822
## Llama 3.1 8b Instruct 0.0398771500 0.035228854 -0.0197506128
## Llama 3 70b Instruct -0.1598791267 0.060928784 0.0069036112
## Llama 3 70b -0.1700470013 0.113544007 0.0603548431
## Llama 3 8b Instruct -0.0528363034 -0.020485786 0.0615905465
## Llama 3 8b 0.1056039590 -0.188110799 0.0014203359
## Mixtral 8x7b Instruct -0.1462858146 -0.114375662 -0.0876017352
## Mixtral 8x7b -0.1190711952 -0.112317444 -0.0789101177
## Olmo 2 32b Dpo 0.0851155165 0.050490147 0.0147718513
## Olmo 2 32b Instruct 0.0778897716 0.041378227 0.0060728445
## Olmo 2 32b Sft 0.1675020634 0.105824554 0.0928305243
## Olmo 2 32b 0.1672888603 0.185138167 0.1490918596
## Olmo 2 1b Dpo -0.2256344037 0.061985353 0.1189521841
## Olmo 2 1b Instruct -0.2119771615 0.051003262 0.1196883408
## Olmo 2 1b Sft -0.2757638164 0.096267196 0.1397967704
## Olmo 2 1b 0.1125006523 0.052144108 0.0660471248
## Olmo 2 13b Dpo 0.1100751350 0.100710970 0.0014700857
## Olmo 2 13b Instruct 0.1104812065 0.097050893 -0.0014515328
## Olmo 2 13b Sft 0.1086600266 0.100604779 -0.0110963348
## Olmo 2 13b 0.0874024115 0.165907277 0.0332077597
## Olmo 2 7b Dpo 0.1967127960 0.260794625 0.2334203673
## Olmo 2 7b Instruct 0.1973062740 0.253317885 0.2224161609
## Olmo 2 7b Sft 0.1542156267 0.264903634 0.2234401826
## Olmo 2 7b 0.0667561238 0.333477467 0.2834512935
## Pythia 1.4b 0.0700190103 -0.078472972 -0.0145604705
## Pythia 12b 0.1057871798 0.161810687 0.1830376054
## Pythia 14m 0.2231353286 -0.303909377 -0.1737310661
## Pythia 160m 0.4974118457 0.223357477 0.2713533881
## Pythia 1b -0.1568561955 0.159131082 0.3334293480
## Pythia 2.8b -0.1439244349 0.066392561 0.1425454876
## Pythia 410m -0.0102713683 0.279965405 0.3649320630
## Pythia 6.9b -0.1803358249 0.225807635 0.2165243477
## Pythia 70m 1.0000000000 0.336347541 0.2162417547
## Qwen 2.5 0.5b Instruct 0.3363475410 1.000000000 0.7878196512
## Qwen 2.5 0.5b 0.2162417547 0.787819651 1.0000000000
## Qwen 2.5 1.5 Instruct 0.1991498842 0.141557939 0.0480478889
## Qwen 2.5 1.5b 0.1635942993 0.143851428 0.0337898789
## Qwen 2.5 14b Instruct 0.0359202151 0.173298606 0.2333692011
## Qwen 2.5 14b 0.0189647090 0.212961014 0.2026544114
## Qwen 2.5 32b Instruct 0.1279911995 0.230856005 0.1361025516
## Qwen 2.5 32b 0.0001579882 0.168703666 0.0187805019
## Qwen 2.5 3b Instruct 0.2288716656 0.172422660 0.1875671386
## Qwen 2.5 3b 0.0606799792 0.173154501 0.1698294833
## Qwen 2.5 72b Instruct 0.0323862465 0.005644175 -0.0306211548
## Qwen 2.5 7b Instruct 0.0129675794 0.131201749 0.1335473026
## Qwen 2.5 7b 0.0175046899 0.121528601 0.1448493636
## Human 0.0770339315 0.099329823 -0.0002780097
## Qwen 2.5 1.5 Instruct Qwen 2.5 1.5b
## Gemma 2 2b Instruct -0.232230721 -0.27679314
## Gemma 2 2b -0.167105293 -0.15158463
## Llama 2 13b Instruct 0.385126424 0.40259524
## Llama 2 13b 0.324765069 0.39802753
## Llama 2 70b Instruct 0.351811557 0.42323816
## Llama 2 70b 0.353140177 0.42266051
## Llama 2 7b Instruct 0.370550193 0.42433092
## Llama 2 7b 0.326524500 0.39397662
## Llama 3.1 70b Instruct 0.072941454 0.16623809
## Llama 3.1 70b 0.170314911 0.24506094
## Llama 3.1 8b Instruct -0.005103341 0.04831793
## Llama 3 70b Instruct -0.028614507 0.06006736
## Llama 3 70b 0.149476251 0.22624467
## Llama 3 8b Instruct -0.197270451 -0.20494325
## Llama 3 8b -0.100168526 -0.16481915
## Mixtral 8x7b Instruct 0.119208583 0.14141271
## Mixtral 8x7b 0.227064651 0.26249235
## Olmo 2 32b Dpo 0.223822257 0.20803906
## Olmo 2 32b Instruct 0.223301563 0.20989615
## Olmo 2 32b Sft 0.230138829 0.21707263
## Olmo 2 32b 0.243974774 0.23338514
## Olmo 2 1b Dpo 0.282683720 0.22915353
## Olmo 2 1b Instruct 0.289992344 0.22242166
## Olmo 2 1b Sft 0.249415485 0.21053343
## Olmo 2 1b -0.029098896 0.02442437
## Olmo 2 13b Dpo 0.178954575 0.21659701
## Olmo 2 13b Instruct 0.181110574 0.21789189
## Olmo 2 13b Sft 0.204805400 0.23554032
## Olmo 2 13b 0.152631531 0.18336028
## Olmo 2 7b Dpo 0.178634680 0.13095480
## Olmo 2 7b Instruct 0.174718738 0.12881585
## Olmo 2 7b Sft 0.154678906 0.10995943
## Olmo 2 7b 0.016664008 0.02143094
## Pythia 1.4b 0.097867094 0.10135684
## Pythia 12b 0.202651588 0.25989029
## Pythia 14m 0.049924767 0.03063976
## Pythia 160m 0.169582139 0.12768845
## Pythia 1b 0.175297682 0.31076788
## Pythia 2.8b 0.034766249 0.09777910
## Pythia 410m 0.281315719 0.32153537
## Pythia 6.9b -0.011743332 -0.01490894
## Pythia 70m 0.199149884 0.16359430
## Qwen 2.5 0.5b Instruct 0.141557939 0.14385143
## Qwen 2.5 0.5b 0.048047889 0.03378988
## Qwen 2.5 1.5 Instruct 1.000000000 0.86914703
## Qwen 2.5 1.5b 0.869147031 1.00000000
## Qwen 2.5 14b Instruct -0.057789896 -0.02214278
## Qwen 2.5 14b 0.048426501 0.10216641
## Qwen 2.5 32b Instruct -0.054839805 -0.11960192
## Qwen 2.5 32b -0.169153031 -0.19359515
## Qwen 2.5 3b Instruct 0.062438371 -0.04111950
## Qwen 2.5 3b 0.045125604 -0.02396310
## Qwen 2.5 72b Instruct -0.091755269 -0.08186557
## Qwen 2.5 7b Instruct 0.061967048 0.13263280
## Qwen 2.5 7b 0.004037591 0.10559245
## Human 0.143355960 0.04600132
## Qwen 2.5 14b Instruct Qwen 2.5 14b Qwen 2.5 32b Instruct
## Gemma 2 2b Instruct -0.038042581 -0.047246281 0.083448171
## Gemma 2 2b -0.217687616 -0.219954634 0.022954507
## Llama 2 13b Instruct 0.030085275 0.174146602 0.068610010
## Llama 2 13b 0.004240204 0.201610773 -0.075367377
## Llama 2 70b Instruct 0.116894747 0.311362524 0.103007783
## Llama 2 70b 0.104327796 0.319445678 -0.013023223
## Llama 2 7b Instruct -0.028440696 0.165129986 -0.046519454
## Llama 2 7b -0.015912838 0.198186239 -0.102408667
## Llama 3.1 70b Instruct 0.402807135 0.508026529 0.323728484
## Llama 3.1 70b 0.540010238 0.647408501 0.340302043
## Llama 3.1 8b Instruct 0.417780336 0.381642020 0.267850469
## Llama 3 70b Instruct 0.514909380 0.580862418 0.458521399
## Llama 3 70b 0.554825486 0.657928411 0.369195122
## Llama 3 8b Instruct 0.126668751 0.013538822 0.001228225
## Llama 3 8b 0.244651345 0.174680264 -0.109201760
## Mixtral 8x7b Instruct 0.217212306 0.263022852 0.127708727
## Mixtral 8x7b 0.207378799 0.313163455 0.114713763
## Olmo 2 32b Dpo 0.551685338 0.551835767 0.283450554
## Olmo 2 32b Instruct 0.546933717 0.539831586 0.285411703
## Olmo 2 32b Sft 0.578740427 0.599503629 0.294181032
## Olmo 2 32b 0.556993876 0.605658867 0.327398042
## Olmo 2 1b Dpo 0.007086175 0.020668205 -0.030043223
## Olmo 2 1b Instruct -0.017389356 0.007877412 -0.072499526
## Olmo 2 1b Sft 0.071088587 0.087526054 0.035085114
## Olmo 2 1b 0.087913540 0.227187565 0.106164288
## Olmo 2 13b Dpo 0.317128137 0.443181543 0.247821559
## Olmo 2 13b Instruct 0.322909278 0.453684768 0.257688254
## Olmo 2 13b Sft 0.258818474 0.379667331 0.199356901
## Olmo 2 13b 0.345771998 0.435602337 0.285343401
## Olmo 2 7b Dpo 0.499305742 0.544240380 0.429257658
## Olmo 2 7b Instruct 0.492117426 0.532551442 0.435263380
## Olmo 2 7b Sft 0.468330448 0.547812065 0.403114449
## Olmo 2 7b 0.394847632 0.400719999 0.398443395
## Pythia 1.4b 0.225932065 0.348172886 0.013803214
## Pythia 12b 0.030831832 0.170523124 0.139309674
## Pythia 14m -0.130379289 -0.042119554 -0.205360317
## Pythia 160m 0.090245569 0.008123214 0.038519162
## Pythia 1b 0.265251740 0.261849086 0.038052770
## Pythia 2.8b 0.142367065 0.293772714 0.069316258
## Pythia 410m 0.028768751 -0.042767890 0.013964043
## Pythia 6.9b 0.148041807 0.304019086 0.182210453
## Pythia 70m 0.035920215 0.018964709 0.127991200
## Qwen 2.5 0.5b Instruct 0.173298606 0.212961014 0.230856005
## Qwen 2.5 0.5b 0.233369201 0.202654411 0.136102552
## Qwen 2.5 1.5 Instruct -0.057789896 0.048426501 -0.054839805
## Qwen 2.5 1.5b -0.022142777 0.102166409 -0.119601923
## Qwen 2.5 14b Instruct 1.000000000 0.803472924 0.476964222
## Qwen 2.5 14b 0.803472924 1.000000000 0.406328360
## Qwen 2.5 32b Instruct 0.476964222 0.406328360 1.000000000
## Qwen 2.5 32b 0.435998255 0.473061187 0.766826850
## Qwen 2.5 3b Instruct 0.154010781 0.081444359 0.321182915
## Qwen 2.5 3b 0.196110723 0.149929965 0.345044956
## Qwen 2.5 72b Instruct 0.298868566 0.284325070 0.465703554
## Qwen 2.5 7b Instruct 0.492594581 0.430099328 0.333801798
## Qwen 2.5 7b 0.477870093 0.410195877 0.354890906
## Human 0.324903977 0.319639141 0.338522262
## Qwen 2.5 32b Qwen 2.5 3b Instruct Qwen 2.5 3b
## Gemma 2 2b Instruct -0.0032701337 0.003695353 -0.205550984
## Gemma 2 2b 0.0125410809 0.005556541 -0.140739366
## Llama 2 13b Instruct 0.0614514455 -0.032110903 0.223711469
## Llama 2 13b -0.0614232689 -0.178827516 0.079176267
## Llama 2 70b Instruct 0.1625479459 -0.052930580 0.195151318
## Llama 2 70b 0.0002291193 -0.153146396 0.071665994
## Llama 2 7b Instruct -0.0366206732 -0.149917176 0.107798227
## Llama 2 7b -0.0881558657 -0.175238436 0.055899940
## Llama 3.1 70b Instruct 0.4680914335 0.024425866 0.159120463
## Llama 3.1 70b 0.4765457698 0.023391030 0.151788753
## Llama 3.1 8b Instruct 0.2757957252 0.059024927 0.304490251
## Llama 3 70b Instruct 0.5855278699 0.054169273 0.204810882
## Llama 3 70b 0.4935688239 0.041436170 0.196311591
## Llama 3 8b Instruct -0.1226472400 0.246351206 0.219186823
## Llama 3 8b -0.1827351005 -0.098105445 -0.112908417
## Mixtral 8x7b Instruct 0.1182892377 0.015955072 0.286495858
## Mixtral 8x7b 0.1088812866 -0.006845552 0.271176411
## Olmo 2 32b Dpo 0.3327618699 0.159321965 0.257103266
## Olmo 2 32b Instruct 0.3302729542 0.163240947 0.259058756
## Olmo 2 32b Sft 0.3345389843 0.184416285 0.254564940
## Olmo 2 32b 0.3741894888 0.195732533 0.273668338
## Olmo 2 1b Dpo 0.0250908581 0.179801433 0.216637287
## Olmo 2 1b Instruct -0.0216689419 0.146527580 0.164310684
## Olmo 2 1b Sft 0.1126888985 0.262349855 0.374894055
## Olmo 2 1b 0.0385937741 0.022651858 -0.027422935
## Olmo 2 13b Dpo 0.3147997025 -0.005355810 0.145559532
## Olmo 2 13b Instruct 0.3245720461 -0.003185889 0.147052815
## Olmo 2 13b Sft 0.2673802264 0.007940175 0.169311726
## Olmo 2 13b 0.3321798848 0.045082532 0.201722726
## Olmo 2 7b Dpo 0.4481293066 0.223652220 0.343651173
## Olmo 2 7b Instruct 0.4546407799 0.223716471 0.345227914
## Olmo 2 7b Sft 0.4336034217 0.227696394 0.334409112
## Olmo 2 7b 0.3086692579 0.205721110 0.390629259
## Pythia 1.4b 0.0511247126 -0.196126790 -0.062993052
## Pythia 12b 0.1093774953 -0.038846725 0.053937509
## Pythia 14m -0.1384457717 -0.059521536 -0.006929151
## Pythia 160m -0.1874265632 0.108877714 0.012089616
## Pythia 1b 0.0110646862 0.028574586 0.237578207
## Pythia 2.8b 0.0624230051 -0.069942124 0.021209940
## Pythia 410m -0.0355586885 0.105746393 0.141399357
## Pythia 6.9b 0.2295646522 0.099726026 0.211100483
## Pythia 70m 0.0001579882 0.228871666 0.060679979
## Qwen 2.5 0.5b Instruct 0.1687036658 0.172422660 0.173154501
## Qwen 2.5 0.5b 0.0187805019 0.187567139 0.169829483
## Qwen 2.5 1.5 Instruct -0.1691530313 0.062438371 0.045125604
## Qwen 2.5 1.5b -0.1935951535 -0.041119500 -0.023963104
## Qwen 2.5 14b Instruct 0.4359982553 0.154010781 0.196110723
## Qwen 2.5 14b 0.4730611866 0.081444359 0.149929965
## Qwen 2.5 32b Instruct 0.7668268504 0.321182915 0.345044956
## Qwen 2.5 32b 1.0000000000 0.203863344 0.287786831
## Qwen 2.5 3b Instruct 0.2038633439 1.000000000 0.797279690
## Qwen 2.5 3b 0.2877868311 0.797279690 1.000000000
## Qwen 2.5 72b Instruct 0.4136721263 0.127055302 0.101872186
## Qwen 2.5 7b Instruct 0.3090646070 0.040086693 0.091829748
## Qwen 2.5 7b 0.2802413536 -0.014016684 -0.046924768
## Human 0.3550362989 0.148927671 0.204228488
## Qwen 2.5 72b Instruct Qwen 2.5 7b Instruct Qwen 2.5 7b
## Gemma 2 2b Instruct 0.238807508 -0.158102063 0.035990949
## Gemma 2 2b 0.256290997 -0.104974321 0.058528393
## Llama 2 13b Instruct 0.125489964 0.254507331 0.147727211
## Llama 2 13b -0.021547644 0.222673386 0.150201931
## Llama 2 70b Instruct 0.032541694 0.338779089 0.239835431
## Llama 2 70b -0.093158634 0.291747368 0.205234127
## Llama 2 7b Instruct -0.069362215 0.178713042 0.071061723
## Llama 2 7b -0.075261306 0.204656240 0.130328971
## Llama 3.1 70b Instruct 0.501295869 0.440635935 0.356036322
## Llama 3.1 70b 0.380845820 0.516588880 0.355407720
## Llama 3.1 8b Instruct 0.314229204 0.459818607 0.460528641
## Llama 3 70b Instruct 0.563641839 0.541649692 0.454414139
## Llama 3 70b 0.364199677 0.541015619 0.391292736
## Llama 3 8b Instruct 0.134932398 -0.168362789 -0.133100532
## Llama 3 8b 0.053596406 -0.006157379 -0.005530676
## Mixtral 8x7b Instruct 0.251367009 0.228321560 0.108692690
## Mixtral 8x7b 0.145834337 0.259136665 0.154419588
## Olmo 2 32b Dpo 0.262027655 0.401192180 0.261336395
## Olmo 2 32b Instruct 0.266573197 0.402293738 0.263540114
## Olmo 2 32b Sft 0.249822040 0.432066249 0.316476101
## Olmo 2 32b 0.256072317 0.412269659 0.295440582
## Olmo 2 1b Dpo -0.143618524 0.159374762 0.090152919
## Olmo 2 1b Instruct -0.165211161 0.128976191 0.069361322
## Olmo 2 1b Sft -0.117443789 0.186568705 0.105574230
## Olmo 2 1b 0.017673172 0.066949770 0.148268886
## Olmo 2 13b Dpo 0.426735077 0.391411239 0.343747097
## Olmo 2 13b Instruct 0.427929209 0.397348385 0.346546033
## Olmo 2 13b Sft 0.379499726 0.361696892 0.309776666
## Olmo 2 13b 0.419267323 0.439806610 0.388440172
## Olmo 2 7b Dpo 0.339718347 0.444248775 0.376342109
## Olmo 2 7b Instruct 0.353747993 0.447338485 0.377616159
## Olmo 2 7b Sft 0.303187316 0.422199877 0.373613055
## Olmo 2 7b 0.277057334 0.393528366 0.331280053
## Pythia 1.4b 0.096137844 0.158092820 0.117395025
## Pythia 12b 0.070779630 0.207307174 0.206241087
## Pythia 14m -0.191349501 -0.118474847 -0.230314239
## Pythia 160m -0.026058025 -0.035566112 -0.069084418
## Pythia 1b -0.014757472 0.344988758 0.260466476
## Pythia 2.8b 0.132414159 0.186515734 0.324455410
## Pythia 410m 0.004869379 0.171168913 0.093188336
## Pythia 6.9b 0.124591830 0.058374501 0.122856364
## Pythia 70m 0.032386247 0.012967579 0.017504690
## Qwen 2.5 0.5b Instruct 0.005644175 0.131201749 0.121528601
## Qwen 2.5 0.5b -0.030621155 0.133547303 0.144849364
## Qwen 2.5 1.5 Instruct -0.091755269 0.061967048 0.004037591
## Qwen 2.5 1.5b -0.081865575 0.132632796 0.105592452
## Qwen 2.5 14b Instruct 0.298868566 0.492594581 0.477870093
## Qwen 2.5 14b 0.284325070 0.430099328 0.410195877
## Qwen 2.5 32b Instruct 0.465703554 0.333801798 0.354890906
## Qwen 2.5 32b 0.413672126 0.309064607 0.280241354
## Qwen 2.5 3b Instruct 0.127055302 0.040086693 -0.014016684
## Qwen 2.5 3b 0.101872186 0.091829748 -0.046924768
## Qwen 2.5 72b Instruct 1.000000000 0.277929423 0.323473632
## Qwen 2.5 7b Instruct 0.277929423 1.000000000 0.814768216
## Qwen 2.5 7b 0.323473632 0.814768216 1.000000000
## Human 0.501441559 0.281365925 0.137389238
## Human
## Gemma 2 2b Instruct 2.892354e-02
## Gemma 2 2b 9.128841e-03
## Llama 2 13b Instruct 2.144244e-01
## Llama 2 13b 1.135046e-01
## Llama 2 70b Instruct 1.841599e-01
## Llama 2 70b 9.306858e-02
## Llama 2 7b Instruct 1.088248e-01
## Llama 2 7b 7.404699e-02
## Llama 3.1 70b Instruct 4.629338e-01
## Llama 3.1 70b 5.172369e-01
## Llama 3.1 8b Instruct 3.078204e-01
## Llama 3 70b Instruct 4.994881e-01
## Llama 3 70b 4.842453e-01
## Llama 3 8b Instruct 2.814842e-02
## Llama 3 8b 5.204098e-02
## Mixtral 8x7b Instruct 3.394477e-01
## Mixtral 8x7b 3.180337e-01
## Olmo 2 32b Dpo 4.926853e-01
## Olmo 2 32b Instruct 4.865321e-01
## Olmo 2 32b Sft 4.736136e-01
## Olmo 2 32b 4.940875e-01
## Olmo 2 1b Dpo -2.940477e-02
## Olmo 2 1b Instruct -4.245679e-02
## Olmo 2 1b Sft -3.787642e-02
## Olmo 2 1b -4.992973e-02
## Olmo 2 13b Dpo 5.083239e-01
## Olmo 2 13b Instruct 5.081724e-01
## Olmo 2 13b Sft 4.867408e-01
## Olmo 2 13b 4.899657e-01
## Olmo 2 7b Dpo 3.816489e-01
## Olmo 2 7b Instruct 3.902405e-01
## Olmo 2 7b Sft 3.151644e-01
## Olmo 2 7b 2.684079e-01
## Pythia 1.4b 7.755252e-02
## Pythia 12b 7.143812e-02
## Pythia 14m -4.276445e-02
## Pythia 160m 4.892553e-02
## Pythia 1b -4.031918e-02
## Pythia 2.8b -3.594982e-02
## Pythia 410m 3.982029e-02
## Pythia 6.9b 8.817533e-05
## Pythia 70m 7.703393e-02
## Qwen 2.5 0.5b Instruct 9.932982e-02
## Qwen 2.5 0.5b -2.780097e-04
## Qwen 2.5 1.5 Instruct 1.433560e-01
## Qwen 2.5 1.5b 4.600132e-02
## Qwen 2.5 14b Instruct 3.249040e-01
## Qwen 2.5 14b 3.196391e-01
## Qwen 2.5 32b Instruct 3.385223e-01
## Qwen 2.5 32b 3.550363e-01
## Qwen 2.5 3b Instruct 1.489277e-01
## Qwen 2.5 3b 2.042285e-01
## Qwen 2.5 72b Instruct 5.014416e-01
## Qwen 2.5 7b Instruct 2.813659e-01
## Qwen 2.5 7b 1.373892e-01
## Human 1.000000e+00# Plot the correlation matrix
ggcorrplot(cor_matrix,
hc.order = FALSE,
method = "square"
)
sort(cor_matrix["Human",])## Olmo 2 1b Pythia 14m Olmo 2 1b Instruct
## -4.992973e-02 -4.276445e-02 -4.245679e-02
## Pythia 1b Olmo 2 1b Sft Pythia 2.8b
## -4.031918e-02 -3.787642e-02 -3.594982e-02
## Olmo 2 1b Dpo Qwen 2.5 0.5b Pythia 6.9b
## -2.940477e-02 -2.780097e-04 8.817533e-05
## Gemma 2 2b Llama 3 8b Instruct Gemma 2 2b Instruct
## 9.128841e-03 2.814842e-02 2.892354e-02
## Pythia 410m Qwen 2.5 1.5b Pythia 160m
## 3.982029e-02 4.600132e-02 4.892553e-02
## Llama 3 8b Pythia 12b Llama 2 7b
## 5.204098e-02 7.143812e-02 7.404699e-02
## Pythia 70m Pythia 1.4b Llama 2 70b
## 7.703393e-02 7.755252e-02 9.306858e-02
## Qwen 2.5 0.5b Instruct Llama 2 7b Instruct Llama 2 13b
## 9.932982e-02 1.088248e-01 1.135046e-01
## Qwen 2.5 7b Qwen 2.5 1.5 Instruct Qwen 2.5 3b Instruct
## 1.373892e-01 1.433560e-01 1.489277e-01
## Llama 2 70b Instruct Qwen 2.5 3b Llama 2 13b Instruct
## 1.841599e-01 2.042285e-01 2.144244e-01
## Olmo 2 7b Qwen 2.5 7b Instruct Llama 3.1 8b Instruct
## 2.684079e-01 2.813659e-01 3.078204e-01
## Olmo 2 7b Sft Mixtral 8x7b Qwen 2.5 14b
## 3.151644e-01 3.180337e-01 3.196391e-01
## Qwen 2.5 14b Instruct Qwen 2.5 32b Instruct Mixtral 8x7b Instruct
## 3.249040e-01 3.385223e-01 3.394477e-01
## Qwen 2.5 32b Olmo 2 7b Dpo Olmo 2 7b Instruct
## 3.550363e-01 3.816489e-01 3.902405e-01
## Llama 3.1 70b Instruct Olmo 2 32b Sft Llama 3 70b
## 4.629338e-01 4.736136e-01 4.842453e-01
## Olmo 2 32b Instruct Olmo 2 13b Sft Olmo 2 13b
## 4.865321e-01 4.867408e-01 4.899657e-01
## Olmo 2 32b Dpo Olmo 2 32b Llama 3 70b Instruct
## 4.926853e-01 4.940875e-01 4.994881e-01
## Qwen 2.5 72b Instruct Olmo 2 13b Instruct Olmo 2 13b Dpo
## 5.014416e-01 5.081724e-01 5.083239e-01
## Llama 3.1 70b Human
## 5.172369e-01 1.000000e+00### MDS
# Convert correlation matrix to a dissimilarity matrix
dissimilarity <- as.dist(sqrt(1 - cor_matrix))
# Perform classical MDS
mds_result <- cmdscale(dissimilarity, k = 2) # k = number of dimensions, usually 2 for plotting
# Put into a data frame for plotting
mds_df <- as.data.frame(mds_result)
colnames(mds_df) <- c("Dim1", "Dim2")
mds_df$model_shorthand <- rownames(mds_df)
mds_df = mds_df %>%
inner_join(df_summ)## Joining with `by = join_by(model_shorthand)`ggplot(mds_df, aes(Dim1, Dim2,
color = model_family,
size = mean_accuracy)) +
geom_point(alpha = .5) +
theme_minimal() +
labs(x = "MDS 1",
y = "MDS 2",
color = "") +
theme(text = element_text(size = 15),
legend.position = "none") +
scale_color_manual(values = viridisLite::viridis(8, option = "mako",
begin = 0.8, end = 0.15)) 
Baselines analysis
First, merge with raw human data.
df_human_shortened = df_human %>%
# mutate(human_is_start = is_start) %>%
select(participant_id, item_id, passage, condition, knowledge_cue,
is_start, reaction_time)
df_all_models_with_human = df_all_models %>%
inner_join(df_human_shortened)## Joining with `by = join_by(passage, knowledge_cue, condition)`## Warning in inner_join(., df_human_shortened): Detected an unexpected many-to-many relationship between `x` and `y`.
## ℹ Row 1 of `x` matches multiple rows in `y`.
## ℹ Row 102 of `y` matches multiple rows in `x`.
## ℹ If a many-to-many relationship is expected, set `relationship =
## "many-to-many"` to silence this warning.table(df_all_models_with_human$model_path)##
## allenai/OLMo-2-0325-32B allenai/OLMo-2-0325-32B-DPO
## 613 613
## allenai/OLMo-2-0325-32B-Instruct allenai/OLMo-2-0325-32B-SFT
## 613 613
## allenai/OLMo-2-0425-1B allenai/OLMo-2-0425-1B-DPO
## 613 613
## allenai/OLMo-2-0425-1B-Instruct allenai/OLMo-2-0425-1B-SFT
## 613 613
## allenai/OLMo-2-1124-13B allenai/OLMo-2-1124-13B-DPO
## 613 613
## allenai/OLMo-2-1124-13B-Instruct allenai/OLMo-2-1124-13B-SFT
## 613 613
## allenai/OLMo-2-1124-7B allenai/OLMo-2-1124-7B-DPO
## 613 613
## allenai/OLMo-2-1124-7B-Instruct allenai/OLMo-2-1124-7B-SFT
## 613 613
## EleutherAI/pythia-1.4b EleutherAI/pythia-12b
## 613 613
## EleutherAI/pythia-14m EleutherAI/pythia-160m
## 613 613
## EleutherAI/pythia-1b EleutherAI/pythia-2.8b
## 613 613
## EleutherAI/pythia-410m EleutherAI/pythia-6.9b
## 613 613
## EleutherAI/pythia-70m google/gemma-2b
## 613 613
## google/gemma-2b-it meta-llama/Llama-2-13b-chat-hf
## 613 613
## meta-llama/Llama-2-13b-hf meta-llama/Llama-2-70b-chat-hf
## 613 613
## meta-llama/Llama-2-70b-hf meta-llama/Llama-2-7b-chat-hf
## 613 613
## meta-llama/Llama-2-7b-hf meta-llama/Llama-3.1-70B
## 613 613
## meta-llama/Llama-3.1-70B-Instruct meta-llama/Llama-3.1-8B-Instruct
## 613 613
## meta-llama/Meta-Llama-3-70B meta-llama/Meta-Llama-3-70B-Instruct
## 613 613
## meta-llama/Meta-Llama-3-8B meta-llama/Meta-Llama-3-8B-Instruct
## 613 613
## mistralai/Mixtral-8x7B-Instruct-v0.1 mistralai/Mixtral-8x7B-v0.1
## 613 613
## Qwen/Qwen2.5-0.5B Qwen/Qwen2.5-0.5B-Instruct
## 613 613
## Qwen/Qwen2.5-1.5B Qwen/Qwen2.5-1.5B-Instruct
## 613 613
## Qwen/Qwen2.5-14B Qwen/Qwen2.5-14B-Instruct
## 613 613
## Qwen/Qwen2.5-32B Qwen/Qwen2.5-32B-Instruct
## 613 613
## Qwen/Qwen2.5-3B Qwen/Qwen2.5-3B-Instruct
## 613 613
## Qwen/Qwen2.5-72B-Instruct Qwen/Qwen2.5-7B
## 613 613
## Qwen/Qwen2.5-7B-Instruct
## 613Now, fit a baselines model for each LLM.
fit_compare_glmer_by_model <- function(df) {
if (n_distinct(df$is_start) < 2) return(NULL) # skip degenerate cases
# Fit full model
mod_full <- tryCatch(
glmer(is_start ~ condition + knowledge_cue +
first_mention + recent_mention + log_odds +
(1 | item_id),
data = df,
family = binomial()),
error = function(e) NULL
)
# Fit reduced model (no condition)
mod_reduced <- tryCatch(
glmer(is_start ~ knowledge_cue +
first_mention + recent_mention + log_odds +
(1 | item_id),
data = df,
family = binomial()),
error = function(e) NULL
)
if (is.null(mod_full) || is.null(mod_reduced)) return(NULL)
# Model comparison (Likelihood Ratio Test)
anova_result <- anova(mod_reduced, mod_full)
lrt_stat <- anova_result$Chisq[2]
p_val <- anova_result$`Pr(>Chisq)`[2]
delta_aic <- AIC(mod_reduced) - AIC(mod_full)
# Extract coefficient for log_odds
log_odds_coef <- tryCatch({
fixef(mod_full)["log_odds"]
}, error = function(e) NA)
tibble(
model_shorthand = unique(df$model_shorthand),
delta_AIC = delta_aic,
LRT_stat = lrt_stat,
p_value = p_val,
log_odds_coef = log_odds_coef,
knowledge_cue_coef = fixef(mod_full)["knowledge_cueImplicit"],
condition_coef = fixef(mod_full)["conditionTrue Belief"],
)
}
# Apply across model_shorthand groups
glmer_model_comparisons <- df_all_models_with_human %>%
group_by(model_shorthand) %>%
group_split() %>%
map_dfr(fit_compare_glmer_by_model)
glmer_model_comparisons## # A tibble: 55 × 7
## model_shorthand delta_AIC LRT_stat p_value log_odds_coef knowledge_cue_coef
## <chr> <dbl> <dbl> <dbl> <dbl> <dbl>
## 1 Gemma 2 2b 190. 192. 1.14e-43 -0.0135 -0.968
## 2 Gemma 2 2b Inst… 191. 193. 9.00e-44 0.00616 -0.944
## 3 Llama 2 13b 191. 193. 7.97e-44 -0.0882 -1.05
## 4 Llama 2 13b Ins… 185. 187. 1.52e-42 -0.0202 -1.01
## 5 Llama 2 70b 191. 193. 5.60e-44 -0.115 -1.01
## 6 Llama 2 70b Ins… 188. 190. 2.97e-43 -0.0940 -1.08
## 7 Llama 2 7b 191. 193. 7.06e-44 -0.0729 -1.01
## 8 Llama 2 7b Inst… 191. 193. 6.79e-44 -0.0559 -1.05
## 9 Llama 3 70b 132. 134. 6.21e-31 -0.123 -1.02
## 10 Llama 3 70b Ins… 135. 137. 1.13e-31 -0.101 -1.28
## # ℹ 45 more rows
## # ℹ 1 more variable: condition_coef <dbl>summary(glmer_model_comparisons$log_odds_coef)## Min. 1st Qu. Median Mean 3rd Qu. Max.
## -0.19328 -0.06085 -0.01341 -0.01413 0.02586 0.12452summary(glmer_model_comparisons$knowledge_cue_coef)## Min. 1st Qu. Median Mean 3rd Qu. Max.
## -1.2764 -1.0142 -0.9798 -0.9794 -0.9439 -0.7799summary(glmer_model_comparisons$condition_coef)## Min. 1st Qu. Median Mean 3rd Qu. Max.
## -4.300 -3.632 -3.590 -3.611 -3.570 -3.274summary(glmer_model_comparisons$LRT_stat)## Min. 1st Qu. Median Mean 3rd Qu. Max.
## 117.6 163.3 187.0 173.2 192.5 195.4PPP Analysis
fit_ppp_by_model <- function(df) {
# Fit full model
mod_full <- tryCatch(
glmer(is_start ~ log_odds+
(1 | item_id),
data = df,
family = binomial()),
error = function(e) NULL
)
# Extract coefficient for log_odds
log_odds_coef <- tryCatch({
fixef(mod_full)["log_odds"]
}, error = function(e) NA)
tibble(
model_shorthand = unique(df$model_shorthand),
AIC = AIC(mod_full),
log_odds_coef = fixef(mod_full)["log_odds"]
)
}
# Apply across model_shorthand groups
ppp_models <- df_all_models_with_human %>%
group_by(model_shorthand) %>%
group_split() %>%
map_dfr(fit_ppp_by_model) %>%
inner_join(df_model_properties)## Warning in checkConv(attr(opt, "derivs"), opt$par, ctrl = control$checkConv, :
## Model failed to converge with max|grad| = 0.104545 (tol = 0.002, component 1)## Warning in checkConv(attr(opt, "derivs"), opt$par, ctrl = control$checkConv, : Model is nearly unidentifiable: very large eigenvalue
## - Rescale variables?## Joining with `by = join_by(model_shorthand)`### Rescale AIC
ppp_models = ppp_models %>%
mutate(delta_AIC = AIC - min(AIC),
weight = exp(-0.5 * delta_AIC),
model_prob = weight / sum(weight))
### Add back to original data
ppp_with_accuracy = ppp_models %>%
select(model_shorthand, model_prob, delta_AIC) %>%
inner_join(df_summ)## Joining with `by = join_by(model_shorthand)`ppp_with_accuracy %>%
summarise(
weighted_accuracy = sum(mean_accuracy * model_prob),
mean_accuracy = mean(mean_accuracy)
)## # A tibble: 1 × 2
## weighted_accuracy mean_accuracy
## <dbl> <dbl>
## 1 0.603 0.573ppp_with_accuracy %>%
ggplot(aes(x = num_training_tokens,
y = delta_AIC,
color = model_family,
shape = base_instruct)) +
geom_point(size = 6,
alpha = .5) +
scale_x_log10() +
geom_text_repel(aes(label=model_shorthand), size=3) +
labs(x = "#Training Tokens",
y = "Delta AIC",
color = "",
shape = "") +
theme_minimal() +
scale_color_manual(values = viridisLite::viridis(8, option = "mako",
begin = 0.8, end = 0.15)) +
theme(text = element_text(size = 15),
legend.position="bottom")## Warning: Removed 2 rows containing missing values or values outside the scale range
## (`geom_point()`).## Warning: Removed 2 rows containing missing values or values outside the scale range
## (`geom_text_repel()`).## Warning: ggrepel: 17 unlabeled data points (too many overlaps). Consider
## increasing max.overlaps
ppp_with_accuracy %>%
ggplot(aes(x = num_params,
y = delta_AIC,
color = model_family,
shape = base_instruct)) +
geom_point(size = 6,
alpha = .5) +
scale_x_log10() +
geom_text_repel(aes(label=model_shorthand), size=3) +
labs(x = "#Parameters",
y = "Delta AIC",
color = "",
shape = "") +
theme_minimal() +
scale_color_manual(values = viridisLite::viridis(8, option = "mako",
begin = 0.8, end = 0.15)) +
theme(text = element_text(size = 15),
legend.position="bottom")## Warning: ggrepel: 21 unlabeled data points (too many overlaps). Consider
## increasing max.overlaps
### Among models that >50% accuracy, better models explain more variance in human data
ppp_with_accuracy %>%
ggplot(aes(x = mean_accuracy,
y = delta_AIC,
color = model_family,
shape = base_instruct)) +
geom_point(size = 6,
alpha = .5) +
scale_x_log10() +
geom_text_repel(aes(label=model_shorthand), size=3) +
labs(x = "Accuracy",
y = "Delta AIC",
color = "",
shape = "") +
theme_minimal() +
scale_color_manual(values = viridisLite::viridis(8, option = "mako",
begin = 0.8, end = 0.15)) +
theme(text = element_text(size = 15),
legend.position="bottom")## Warning: ggrepel: 24 unlabeled data points (too many overlaps). Consider
## increasing max.overlaps
mod_ppp = lmer(data = ppp_with_accuracy,
delta_AIC ~ # mean_accuracy +
log10(num_params) + log10(num_training_tokens)+
(1 | model_family))
summary(mod_ppp)## Linear mixed model fit by REML. t-tests use Satterthwaite's method [
## lmerModLmerTest]
## Formula: delta_AIC ~ log10(num_params) + log10(num_training_tokens) +
## (1 | model_family)
## Data: ppp_with_accuracy
##
## REML criterion at convergence: 437.5
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -2.0956 -0.7921 0.1029 0.7594 1.6498
##
## Random effects:
## Groups Name Variance Std.Dev.
## model_family (Intercept) 231.8 15.23
## Residual 253.4 15.92
## Number of obs: 53, groups: model_family, 7
##
## Fixed effects:
## Estimate Std. Error df t value Pr(>|t|)
## (Intercept) 439.481 130.930 4.280 3.357 0.0256 *
## log10(num_params) -23.728 3.739 49.009 -6.346 6.83e-08 ***
## log10(num_training_tokens) -11.785 10.728 4.602 -1.099 0.3261
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) l10(_)
## lg10(nm_pr) 0.001
## lg10(nm_t_) -0.961 -0.273Knowledge cue
mod_full = glmer(is_start ~ condition + knowledge_cue +
first_mention + recent_mention +
(1 | item_id),
data = df_human,
family = binomial())
mod_no_kc = glmer(is_start ~ condition + # knowledge_cue +
first_mention + recent_mention +
(1 | item_id),
data = df_human,
family = binomial())
anova(mod_full, mod_no_kc)## Data: df_human
## Models:
## mod_no_kc: is_start ~ condition + first_mention + recent_mention + (1 | item_id)
## mod_full: is_start ~ condition + knowledge_cue + first_mention + recent_mention + (1 | item_id)
## npar AIC BIC logLik -2*log(L) Chisq Df Pr(>Chisq)
## mod_no_kc 5 548.97 571.07 -269.49 538.97
## mod_full 6 534.97 561.48 -261.48 522.97 16.008 1 6.308e-05 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1summary(mod_full)## Generalized linear mixed model fit by maximum likelihood (Laplace
## Approximation) [glmerMod]
## Family: binomial ( logit )
## Formula:
## is_start ~ condition + knowledge_cue + first_mention + recent_mention +
## (1 | item_id)
## Data: df_human
##
## AIC BIC logLik -2*log(L) df.resid
## 535.0 561.5 -261.5 523.0 607
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -4.1452 -0.4658 0.2244 0.3590 2.5435
##
## Random effects:
## Groups Name Variance Std.Dev.
## item_id (Intercept) 0.1658 0.4072
## Number of obs: 613, groups: item_id, 185
##
## Fixed effects:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) 2.62696 0.32947 7.973 1.55e-15 ***
## conditionTrue Belief -3.58666 0.31800 -11.279 < 2e-16 ***
## knowledge_cueImplicit -0.96220 0.24941 -3.858 0.000114 ***
## first_mentionStart 0.01673 0.23516 0.071 0.943277
## recent_mentionStart 0.32950 0.23738 1.388 0.165115
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) cndtTB knwl_I frst_S
## condtnTrBlf -0.691
## knwldg_cImp -0.560 0.330
## frst_mntnSt -0.368 0.004 -0.029
## rcnt_mntnSt -0.299 -0.127 0.015 0.025### Comopare to parameter estimate for LLMs, using binary outcome
df_all_models$is_start = df_all_models$log_odds > 0
mod_all_lms_categorical = glmer(is_start ~ condition + knowledge_cue +
first_mention + recent_mention +
(1 + condition | model_path) + (1 | start),
data = df_all_models,
family = binomial())
summary(mod_all_lms_categorical)## Generalized linear mixed model fit by maximum likelihood (Laplace
## Approximation) [glmerMod]
## Family: binomial ( logit )
## Formula:
## is_start ~ condition + knowledge_cue + first_mention + recent_mention +
## (1 + condition | model_path) + (1 | start)
## Data: df_all_models
##
## AIC BIC logLik -2*log(L) df.resid
## 12104.1 12169.4 -6043.0 12086.1 10551
##
## Scaled residuals:
## Min 1Q Median 3Q Max
## -9.3177 -0.7620 0.1491 0.7755 3.9319
##
## Random effects:
## Groups Name Variance Std.Dev. Corr
## model_path (Intercept) 2.373 1.5404
## conditionTrue Belief 1.201 1.0957 -0.89
## start (Intercept) 0.207 0.4549
## Number of obs: 10560, groups: model_path, 55; start, 10
##
## Fixed effects:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) 1.05182 0.25949 4.053 5.05e-05 ***
## conditionTrue Belief -0.94348 0.15690 -6.013 1.82e-09 ***
## knowledge_cueImplicit -0.73516 0.04498 -16.343 < 2e-16 ***
## first_mentionStart 0.01097 0.04457 0.246 0.8056
## recent_mentionStart 0.09870 0.04457 2.214 0.0268 *
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Correlation of Fixed Effects:
## (Intr) cndtTB knwl_I frst_S
## condtnTrBlf -0.719
## knwldg_cImp -0.094 0.017
## frst_mntnSt -0.086 0.000 -0.001
## rcnt_mntnSt -0.084 -0.002 -0.008 0.000coef_human <- fixef(mod_full)
coef_llms <- fixef(mod_all_lms_categorical)
# Create aligned dataframe
group_coefs <- tibble(
model_path = c("Human", "LLMs"),
knowledge_cue_coef = c(coef_human["knowledge_cueImplicit"], coef_llms["knowledge_cueImplicit"]),
condition_coef = c(coef_human["conditionTrue Belief"], coef_llms["conditionTrue Belief"])
)
## Each model on its own
run_models_for_comparison <- function(df) {
mod_full = glmer(is_start ~ condition + knowledge_cue +
first_mention + recent_mention +
(1 + condition | start),
data = df,
family = binomial(),
control = glmerControl(optimizer = "bobyqa"))
# Extract coefficients
coefs <- fixef(mod_full)
cond_coef <- coefs[grep("^condition", names(coefs))]
cue_coef <- coefs[grep("^knowledge_cue", names(coefs))]
tibble(
model_path = unique(df$model_path),
condition_coef = cond_coef,
knowledge_cue_coef = cue_coef
)
}
# Apply to each model_path
results_by_model_path <- df_all_models %>%
group_by(model_path) %>%
group_split() %>%
map_dfr(run_models_for_comparison)## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')## Warning in checkConv(attr(opt, "derivs"), opt$par, ctrl = control$checkConv, :
## Hessian is numerically singular: parameters are not uniquely determined## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')
## boundary (singular) fit: see help('isSingular')## Warning in checkConv(attr(opt, "derivs"), opt$par, ctrl = control$checkConv, : Model is nearly unidentifiable: large eigenvalue ratio
## - Rescale variables?## boundary (singular) fit: see help('isSingular')## Warning in checkConv(attr(opt, "derivs"), opt$par, ctrl = control$checkConv, : Model is nearly unidentifiable: large eigenvalue ratio
## - Rescale variables?## Warning in checkConv(attr(opt, "derivs"), opt$par, ctrl = control$checkConv, :
## unable to evaluate scaled gradient## Warning in checkConv(attr(opt, "derivs"), opt$par, ctrl = control$checkConv, :
## Model failed to converge: degenerate Hessian with 1 negative eigenvalues## Warning in checkConv(attr(opt, "derivs"), opt$par, ctrl = control$checkConv, :
## unable to evaluate scaled gradient## Warning in checkConv(attr(opt, "derivs"), opt$par, ctrl = control$checkConv, :
## Hessian is numerically singular: parameters are not uniquely determinedall_coefs = results_by_model_path %>%
select(model_path, knowledge_cue_coef, condition_coef) %>%
bind_rows(group_coefs)
all_coefs_long = all_coefs %>%
pivot_longer(cols = c(knowledge_cue_coef, condition_coef),
names_to = "Term",
values_to = "Estimate") %>%
mutate(Term = fct_recode(Term,
"Knowledge Cue (Implicit)" = "knowledge_cue_coef",
"Condition (True Belief)" = "condition_coef"))
reference_coefs <- all_coefs_long %>%
filter(model_path %in% c("Human", "LLMs"))
all_coefs_long %>%
filter(model_path != "Human") %>%
filter(model_path != "LLMs") %>%
ggplot(aes(x = Term,
y = Estimate)) +
geom_jitter(width = .05, alpha = .3) +
geom_point(data = reference_coefs,
aes(x = Term, y = Estimate, color = model_path),
size = 6,
alpha = .8,
shape = 18,
inherit.aes = FALSE) +
theme_minimal() +
geom_hline(yintercept = 0, linetype = "dotted") +
theme(
legend.position = "bottom"
) +
labs(x = "",
color = "") +
coord_flip() +
theme(axis.title = element_text(size=rel(1.2)),
axis.text = element_text(size = rel(1.2)),
legend.text = element_text(size = rel(1.2)),
legend.title = element_text(size = rel(1.2)),
strip.text.x = element_text(size = rel(1.2))) +
scale_color_manual(values = viridisLite::viridis(2, option = "mako",
begin = 0.8, end = 0.15))
all_coefs_long %>%
arrange(Estimate) %>%
head(3)## # A tibble: 3 × 3
## model_path Term Estimate
## <chr> <fct> <dbl>
## 1 google/gemma-2b-it Condition (True Belief) -25.8
## 2 meta-llama/Meta-Llama-3-70B Condition (True Belief) -22.1
## 3 meta-llama/Llama-3.1-70B Condition (True Belief) -19.8